Skip to content
Snippets Groups Projects

Unified conversion script

  • Clone with SSH
  • Clone with HTTPS
  • Embed
  • Share
    The snippet can be accessed without any authentication.
    Authored by Phillip Thurston

    Script to make sure everything is set when converting a repo from old legacy structures to our new unified k8s and serverless structure.

    Run with:

    bash <(curl -s https://git.netzilla.io/-/snippets/26/raw)

    or to clear npm modules, lock and install/build run with:

    bash <(curl -s https://git.netzilla.io/-/snippets/26/raw) fresh

    Edited
    unifiedConversionScript.sh 20.16 KiB
    #!/bin/bash
    
    # This is a script designed to help getting existing projects to work with our 
    # new unified starter and deployments.
    
    function wordShouldNotExist {
      if [[ $(grep -ir --exclude-dir=node_modules --exclude-dir=.git "$1" ./) ]]; then
        echo -e "\e[91mThe word '$1' is present in the project\e[0m"
        grep -ir --exclude-dir=node_modules --exclude-dir=.git "$1" ./
        export CONFIG_CHECK_FAIL=true
      else
        echo -e "[\e[92mSUCCESS\e[0m] the word '$1' is not in the project"
      fi
    }
    
    function wordShouldExist {
      if [[ $(grep -ir --exclude-dir=node_modules --exclude-dir=.git "$1" ./) ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] the word '$1' is in the project"
      else
        echo -e "\e[91mThe word '$1' is not present in the project\e[0m"
        grep -ir --exclude-dir=node_modules --exclude-dir=.git "$1" ./
        export CONFIG_CHECK_FAIL=true
      fi
    }
    
    # If run with argument "fresh" it will clear npm modules lock and then reinstall and build
    if [[ $1 == "fresh" ]]; then
      echo -e "Removing node_modules/ dir in current directory"
      rm -rf node_modules
      echo -e "Removing package-lock.json in current directory"
      rm package-lock.json
      echo -e "Installing NPM modules and running build"
      set -e
      npm install
      npm run build
      set +e
    fi
    
    if [ $(grep -oP 'PROD_SUBDOMAIN: \K[a-z\-]+' .gitlab-ci.yml 2>/dev/null) ]; then
      PROD_SUBDOMAIN=$(grep -oP 'PROD_SUBDOMAIN: \K[a-z\-]+' .gitlab-ci.yml)
      echo -e "prod subdomain is set as \"$PROD_SUBDOMAIN\""
    fi
    if [ $(grep -oP '^  SERVERLESS_ENV: \K[a-z\w]+' .gitlab-ci.yml 2>/dev/null) ]; then
      SERVERLESS_ENV=$(grep -oP '^  SERVERLESS_ENV: \K[a-z\w]+' .gitlab-ci.yml)
      echo -e "ENV is set as \"$SERVERLESS_ENV\""
    fi
    if [[ $(grep LEGACY_DEPLOY .gitlab-ci.yml 2>/dev/null) ]]; then
      LEGACY_DEPLOY='true'
      echo -e "Legacy Deploy detected"
    fi
    if [[ $(grep NO_SERVERLESS_ENDPOINT .gitlab-ci.yml 2>/dev/null) ]]; then
      NO_SERVERLESS_ENDPOINT='true'
      echo -e "Scheduled lambda detected"
    fi
    if [[ $(grep NPM_PACKAGE_PUBLISH .gitlab-ci.yml 2>/dev/null) ]]; then
      NPM_PACKAGE_PUBLISH='true'
      echo -e "NPM repo project detected"
    fi
    
    
    CI_PROJECT_URL='https://test.com'
    CI_COMMIT_REF_SLUG='testbranch'
    
    if [[ ! -z ${SERVERLESS_ENV+x} ]]; then
      if [[ -f "./serverless.yml" ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] The serverless.yml exists"
      else
        echo -e "\e[91mThe serverless Configuration is missing!\e[0m The serverless.yml file is missing. Is this a serverless project? Please check the lambda starter template for examples"
        exit 1
      fi
      if [[ -f "./package.json" ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] The package.json exists"
      else
        echo -e "\e[91mThe serverless Configuration is invalid!\e[0m The package.json file is missing. Is this a node project? Please check the lambda starter template for examples"
        exit 1
      fi
      if [[ ! $(grep 'junit.xml' ./.gitignore) ]]; then
        echo -e "\e[91mjunit.xml isn't in the gitignore\e[0m"
        export CONFIG_CHECK_FAIL=true
      else
        echo -e "[\e[92mSUCCESS\e[0m] jest results are in the gitignore"
      fi
      if [[ -z ${NO_SERVERLESS_ENDPOINT+x} ]]; then
        if [ -z ${PROD_SUBDOMAIN+x} ]; then
          echo -e "\e[91mThe PROD_SUBDOMAIN variable is unset!\e[0m This script is designed to be run with gitlab CI/CD Jobs"
          export CONFIG_CHECK_FAIL=true
        fi
        if [[ $PROD_SUBDOMAIN == "changeme" ]]; then
          echo -e "\e[91mThe Prod subdomain is still set to it's default!\e[0m You must change the project's PROD_SUBDOMAIN variable to a unqiue subdomain in the .gitlab-ci.yml file"
          export CONFIG_CHECK_FAIL=true
        elif [[ -z ${PROD_SUBDOMAIN+x} ]]; then
          echo -e "\e[91mThe test cannot run!\e[0m because the PROD_SUBDOMAIN is not set!"
          export CONFIG_CHECK_FAIL=true
        else
          echo -e "[\e[92mSUCCESS\e[0m] The Production subdomain appears to be valid and is: $PROD_SUBDOMAIN"
        fi
        if [[ $(echo $PROD_SUBDOMAIN | grep --perl-regexp [^a-z0-9\-]) ]]; then
          echo -e "\e[91mThe Prod subdomain variable is formatted improperly or like a subzone!\e[0m You must change the PROD_SUBDOMAIN variable in the .gitlab-ci.yml file to be a valid subdomain. Containing only (lowercase) a-z, 0-9, or hyphens (-)"
          export CONFIG_CHECK_FAIL=true
        elif [[ -z ${PROD_SUBDOMAIN+x} ]]; then
          echo -e "\e[91mThe test cannot run!\e[0m because the PROD_SUBDOMAIN is not set!"
          export CONFIG_CHECK_FAIL=true
        else
          echo -e "[\e[92mSUCCESS\e[0m] The Production subdomain appears to be RFC compliant"
        fi
        if [ $(echo $PROD_SUBDOMAIN | grep --perl-regexp "\-?stage\-?") ] || [ $(echo $PROD_SUBDOMAIN | grep --perl-regexp "\-?dev\-?") ]; then
          echo -e "\e[91mThe Prod subdomain variable is is confusing\e[0m You must change the PROD_SUBDOMAIN variable in the .gitlab-ci.yml file to not have the words stage or dev in it since our automatically generated staging and dev URLS contain those words and we don't want confusion."
          export CONFIG_CHECK_FAIL=true
        elif [[ -z ${PROD_SUBDOMAIN+x} ]]; then
          echo -e "\e[91mThe test cannot run!\e[0m because the PROD_SUBDOMAIN is not set!"
          export CONFIG_CHECK_FAIL=true
        else
          echo -e "[\e[92mSUCCESS\e[0m] The Production subdomain appears to play nice with our automation"
        fi
        if [[ $(grep 'serverless-domain-manager' ./serverless.yml) ]]; then
          echo -e "[\e[92mSUCCESS\e[0m] The serverless domain manager plugin seems to be in the config"
        else
          echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the serverless-domain-manager plugin is installed. Please check the lambda starter template for examples"
          export CONFIG_CHECK_FAIL=true
        fi
        if [[ $(grep "production: \${file(\.\/\.gitlab-ci\.yml):variables\.PROD_SUBDOMAIN}\.\${env:CERTIFICATE_DOMAIN, 'localhost'}" ./serverless.yml) ]]; then
          echo -e "[\e[92mSUCCESS\e[0m] The production URI is properly referenced in the config"
        else
          echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the production domain variable is properly formatted in the serverless.yml file. Please check the lambda starter template for examples"
          export CONFIG_CHECK_FAIL=true
        fi
        if [[ $(grep "\${file(\.\/\.gitlab-ci\.yml):variables\.PROD_SUBDOMAIN}-stage0\.\${env:CERTIFICATE_DOMAIN, 'localhost'}" ./serverless.yml) ]]; then
          echo -e "[\e[92mSUCCESS\e[0m] The staging URI is properly referenced in the config"
        else
          echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the staging domain variable is properly formatted in the serverless.yml file. Please check the lambda starter template for examples"
          export CONFIG_CHECK_FAIL=true
        fi
        if [[ $(grep 'serverless-domain-manager' ./package.json) ]]; then
          echo -e "[\e[92mSUCCESS\e[0m] Serverless Domain Manager seems to be installed in the package.json"
        else
          echo -e "\e[91mThere are missing npm packages!\e[0m Serverless Domain Manager doesn't seem to be installed. To do so simply run 'npm install serverless-domain-manager --save-dev'. This results in having automatic domains in serverless"
          export CONFIG_CHECK_FAIL=true
        fi
      fi
      if [[ $(grep 'serverless-domain-manager' ./serverless.yml) && ! -z ${NO_SERVERLESS_ENDPOINT+x} ]]; then
        echo -e "\e[91mThe serverless Configuration out of order!\e[0m It seems that domains are configured for this project even though it is listed as having no active endpoints. If this project is going to be unreachable to the outside world please cleanup the serverless.yml and package.json. If any part of the project will be accessed via a URI then make sure the NO_SERVERLESS_ENDPOINT variable is unset."
        export CONFIG_CHECK_FAIL=true
      else
        echo -e "[\e[92mSUCCESS\e[0m] There don't seem to be any conflicting endpoint configurations."
      fi
      if [[ -z ${LEGACY_DEPLOY+x} ]]; then
        if [[ $(grep "service: \${self:custom.env}*" serverless.yml) && -z ${LEGACY_DEPLOY+x} ]]; then
          echo -e "[\e[92mSUCCESS\e[0m] The env opt is part of the service name"
          if [[ $(grep "service:*" serverless.yml | sed -r 's/service:\s\$\{self:custom\.env\}(.*$)/\1/' | wc -c) -le 18 ]]; then
            echo -e "[\e[92mSUCCESS\e[0m] The service name isn't too long."
          else
            echo -e "\e[91mThe deployment will fail!\e[0m AWS (specifically AmazonIdentityManagement) requires that all names are shorter than 64 characters. With how our selected variables and settings are configured our names can't be longer than 18 characters AFTER the \${self:custom.env}.\nThe service name computed ( $(grep "service:*" serverless.yml | sed -r 's/service:\s\$\{self:custom\.env\}(.*$)/\1/') ) counted out is $(grep "service:*" serverless.yml | sed -r 's/service:\s\$\{self:custom\.env\}(.*$)/\1/' | wc -c) characters long. Make sure it's 18 or less characters"
            export CONFIG_CHECK_FAIL=true
          fi
          if [[ $(grep "service: \${self:custom.env}-changeme" serverless.yml) ]]; then
            echo -e "\e[91mThe service name is still set to it's default!\e[0m You must change the project's service name to a unqiue name in the serverless.yml file. The default is \"\${self:custom.env}-changeme\" and it should be changed to something like \"\${self:custom.env}-newproject\""
            export CONFIG_CHECK_FAIL=true
          else
            echo -e "[\e[92mSUCCESS\e[0m] The service name is not the default."
          fi
        else
          echo -e "\e[91mThe deployment is not future proof!\e[0m all deployments must have unique names so if we ever add any additional env's or deployments for this project we need some way to make sure the service name is unique. Make sure you have something like \"service: \${self:custom.env}-example\" listed in the serverless.yml even if we never plan to make this a multi-env project. We want to stay consistent. Feel free to check out our starter for more examples."
          export CONFIG_CHECK_FAIL=true
        fi
      fi
      if [[ $(grep 'deploymentBucket' ./serverless.yml) ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] The deployment bucket seems to be referenced in the config"
      else
        echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the the deploymentBucket is set to our centralized s3 bucket. Please check the lambda starter template for examples"
        export CONFIG_CHECK_FAIL=true
      fi
      if [[ $(grep 'logRetentionInDays' ./serverless.yml) ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] Log retention seems to be referenced in the config"
      else
        echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the the logRetentionInDays is set to a sane value. Please check the lambda starter template for examples"
        export CONFIG_CHECK_FAIL=true
      fi
      if [[ $(grep --perl-regexp '[\#[:space:]]+profile' ./serverless.yml) ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] AWS creds will use the default profile"
      else
        echo -e "\e[91mThe serverless Configuration is invalid!\e[0m Whoops, our CI uses the default AWS profile. Make sure you comment out the 'provider.profile' section of the serverless.yml. If you're unsure how to do this reference the lambda starter."
        export CONFIG_CHECK_FAIL=true
      fi
      if [[ $(grep 'service: serverless-example' ./serverless.yml) ]]; then
        echo -e "\e[91mThe service name is still set to it's default!\e[0m You must make sure the 'service' parameter of the serverless.yml is not set to its default. Please change it to something unique to all our other projects."
        export CONFIG_CHECK_FAIL=true
      else
        echo -e "[\e[92mSUCCESS\e[0m] The serverless service name has been changed from the default."
      fi
      if [ -z ${COMPANY_NAME+x} ] && [ -z ${SERVERLESS_ENV+x} ]; then
        echo -e "\e[91mNo companies/envs are defined!\e[0m Our automation either looks for a list of companies to use (SERVERLESS_ENV) or a single company to use (SERVERLESS_ENV). If the projet  is multi-tenanted with multiple ENVS then use the SERVERLESS_ENV other wise for a deployment only needed by a single company use COMPANY_NAME. One of these two variables must be defined."
        export CONFIG_CHECK_FAIL=true
      else
        echo -e "[\e[92mSUCCESS\e[0m] A company seems to have been configured."
      fi
      if [ ! -z ${COMPANY_NAME+x} ] && [ ! -z ${SERVERLESS_ENV+x} ]; then
        echo -e "\e[91mThe COMPANY_NAME variable and SERVERLESS_ENV are both defined!\e[0m Our automation either looks for a list of companies to use (SERVERLESS_ENV) or a single company to use (SERVERLESS_ENV). If the projet  is multi-tenanted with multiple ENVS then use the SERVERLESS_ENV other wise for a deployment only needed by a single company use COMPANY_NAME. Do not use both variables at the same time."
        export CONFIG_CHECK_FAIL=true
      else
        echo -e "[\e[92mSUCCESS\e[0m] Only one list of companies is provided."
      fi
      if [[ ! -z ${SERVERLESS_ENV+x} ]]; then
        TEST_ENV=( $SERVERLESS_ENV )
        CONTROL_ENV=( blockchainjedi blockchainjedi connect connect codex codex green green blue blue give give ) # Gross https://stackoverflow.com/a/28161520/9879357
        INVALID_ENV=(`echo ${TEST_ENV[@]} ${CONTROL_ENV[@]} | tr ' ' '\n' | sort | uniq -u `)
        if [ ! -z ${INVALID_ENV+x} ]; then
          echo -e "\e[91mThe SERVERLESS_ENV variable has an unknown element!\e[0m You may be adding a brand new type of ENV. that's fine, just have DevOps quicly add it to the automation or use the single target deployment seen in the lambda starter repo. The accepted values for the SERVERLESS_ENV varible are\n=====\n- blockchainjedi \n- connect \n- codex \n- green \n- blue \n- give\n====="
          export CONFIG_CHECK_FAIL=true
        else
          echo -e "[\e[92mSUCCESS\e[0m] The listed server enviroments are all supported"
        fi
          if [[ $(grep 'CI_PROJECT_URL' ./serverless.yml) ]]; then
            echo -e "[\e[92mSUCCESS\e[0m] The serverless functions seem to have their descriptions"
          else
            echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure each function has a description that contains our \${env:CI_PROJECT_URL}. This is important for management for dynamically deployed functions. Please check the lambda starter template for examples"
            export CONFIG_CHECK_FAIL=true
          fi
    
        if [[ $SERVERLESS_ENV =~ [A-Z] ]]; then
          echo -e "\e[91mThe SERVERLESS_ENV variable should have no capitalization!\e[0m make sure that your listed ENVS are all lowercase."
          export CONFIG_CHECK_FAIL=true
        else
          echo -e "[\e[92mSUCCESS\e[0m] The listed server enviroments are all lowercase"
        fi
      fi
      if [[ $(grep '\"test\": \"jest --watch\"' ./package.json) || $(grep '\"test\": \"jest --watchAll\"' ./package.json) ]]; then
        echo -e "[\e[91mERROR\e[0m] Jest is set to 'watch' this means that the job will never complete. Make sure you have 'jest --coverage' somewhere in the 'test' script of your package.json."
        export CONFIG_CHECK_FAIL=true
      fi
      if [[ $(grep 'jest-junit' ./package.json) ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] JUnit reports will be used and reported to the git system for easy analysis."
      else
        echo -e "[\e[93mWARNING\e[0m] JUnit reporting doesn't seem to be enabled. To do so simply run 'npm install jest-junit --save-dev' and add jest-junit to the reporters array in the jest config. this results in being able to see test results directly in merge requests."
      fi
      if [[ $(grep 'cobertura' ./jest.config.js) ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] Cobertura reports should be generated to help the git system show untested lines in git diffs."
      else
        echo -e "[\e[93mWARNING\e[0m] Cobertura reporting doesn't seem to be enabled. To do so simply add 'cobertura' to the coverageReporters array in the jest config."
      fi
      if [[ ! $(grep '\"test\": \"jest --coverage\"' ./package.json) ]]; then
        echo -e "[\e[93mWARNING\e[0m] Jest didn't output any proper coverage reporting. Make sure you have 'jest --coverage' somewhere in the 'test' script of your package.json. From that point on you should see code coverage reports with pipelines in the git system"
      fi
      if [[ $(grep 'deploy:prod' ./package.json) ]]; then
        echo -e "[\e[93mWARNING\e[0m] deploy scripts are still in the package.json"
      fi
      if [[ ! -z ${LEGACY_DEPLOY+x} && ! $(grep -i 'warning' ./.gitlab-ci.yml) ]]; then
        echo -e "[\e[93mWARNING\e[0m] Legacy warning message is missing"
      fi
    fi
    
    if [[ $(grep -oP 'COMPANIES: \K[a-z\-]+' .gitlab-ci.yml 2>/dev/null) ]]; then
      echo -e "[\e[92mSUCCESS\e[0m] The COMPANIES env variable is declared"
    else
      echo -e "[\e[91mERROR\e[0m] The COMPANIES env variable is unset. You must have that variable declared."
      CONFIG_CHECK_FAIL=true
    fi
    
    
    
    if [[ -f package.json ]]; then
      if [[ -z ${NPM_PACKAGE_PUBLISH+x} ]]; then
        if [[ -f .npmrc ]]; then
          echo -e "[\e[92mSUCCESS\e[0m] the .npmrc file exists."
    
          if [[ $(grep '\@blockbrothers:registry=https:\/\/npm\.connectblockchain\.net' .npmrc) ]]; then
            echo -e "[\e[92mSUCCESS\e[0m] the BB NPM registry is in the .npmrc file"
          else
            echo -e "\e[91mThe BB NPM registry isn't properly listed in the project's .npmrc file!\e[0m You must make sure the .npmrc file exists and has the bb registry inside of it."
            export CONFIG_CHECK_FAIL=true
          fi
        else
          echo -e "\e[91mThe .npmrc file is missing!\e[0m You must make sure the .npmrc file exists and has the bb registry inside of it."
          export CONFIG_CHECK_FAIL=true
        fi
      fi
    
      if [[ -x .husky/pre-commit ]]; then
        echo -e "[\e[92mSUCCESS\e[0m] the the husky version bump script is present and executable"
      else
        echo -e "[\e[91mError\e[0m] The husky version bump script is either not present or executable"
        export CONFIG_CHECK_FAIL=true
      fi
    
      if [[ $(grep '\"prepublish\"' package.json) ]]; then
        echo -e "\e[91mprepublish is used in in the package.json file!\e[0m using the prepublish script in NPM scripts is not advisable since it will run those commands on any install command including on prod servers. Instead replace it with prepublishOnly which will only run those commands before you publish your NPM script and not on every install."
        export CONFIG_CHECK_FAIL=true
      fi
    fi
    
    if [[ ! $(grep -i 'Phillip Thurston' Dockerfile) ]]; then
      echo -e "[\e[91mError\e[0m] It looks like the dockerfile hasn't been setup yet."
      export CONFIG_CHECK_FAIL=true
    else
      echo -e "[\e[92mSUCCESS\e[0m] the Dockerfile has been setup"
    fi
    
    if [[ ! $(grep -i '\"dockerport\": 3000' package.json) ]]; then
      echo -e "[\e[91mError\e[0m] It looks like the dockerport config isn't in the package.json."
      export CONFIG_CHECK_FAIL=true
    else
      echo -e "[\e[92mSUCCESS\e[0m] The dockerport is configured"
    fi
    
    if [[ ! $(grep -i 'npm_package_config_dockerport:3000' package.json) ]]; then
      echo -e "[\e[91mError\e[0m] It looks like the Docker run command isn't using a good port."
      export CONFIG_CHECK_FAIL=true
    else
      echo -e "[\e[92mSUCCESS\e[0m] Docker run is using the correct ports"
    fi
    
    if [[ ! $(grep -i 'EXPOSE 3000' Dockerfile) ]]; then
      echo -e "[\e[91mError\e[0m] Correct port is set in the dockerfile"
      export CONFIG_CHECK_FAIL=true
    else
      echo -e "[\e[92mSUCCESS\e[0m] The dockerfile is either not using a port or using an invalid port"
    fi
    
    if [[ ! $(grep -i '\"husky install\"' package.json) ]]; then
      echo -e "[\e[91mError\e[0m] the version bump script hook isn't in the package.json"
      export CONFIG_CHECK_FAIL=true
    else
      echo -e "[\e[92mSUCCESS\e[0m] The version bump script git hook is active."
    fi
    
    if [[ -f .travis.yml ]]; then
      echo -e "[\e[91mError\e[0m] There is a travis file in this project. We don't use travis"
      export CONFIG_CHECK_FAIL=true
    fi
    
    if [[ -f .dockerignore ]]; then
      echo -e "[\e[92mSUCCESS\e[0m] .dockerignore is present"
    else
      echo -e "[\e[91mError\e[0m] The .dockerignore file is not present"
      export CONFIG_CHECK_FAIL=true
    fi
    
    if [[ $(grep -ir --exclude-dir=node_modules --exclude-dir=.git "/health" ./) ]]; then
      echo -e "[\e[92mSUCCESS\e[0m] There seems to be a healthcheck"
    else
      echo -e "[\e[91mError\e[0m] There doesn't seem to be a healthcheck api endpoint. make sure we have a /health endpoint that at minimum responds with a 200 GET request"
      export CONFIG_CHECK_FAIL=true
    fi
    
    wordShouldNotExist starter
    wordShouldNotExist image:build
    wordShouldNotExist loyalty # Keyword check in package.json
    
    wordShouldExist MONGODB_URI_ # Make sure we are using the new mongo FORMAT
    
    
    if [ ! -z ${CONFIG_CHECK_FAIL+x} ]; then
      echo -e "Quick tips:\ngrep -ir --exclude-dir=node_modules --exclude-dir=.git \"<i>\" ./"
      exit 1
    fi
    
    echo -e "\e[91mRemove the Fork relationship in the project\e[0m"
    echo -e "\e[91mSet MR to fail if pipeline isn't successful\e[0m"
    echo -e "\e[91mSet tags to protected to prevent prod pushes\e[0m"
    
    0% Loading or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment