Unified conversion script
The snippet can be accessed without any authentication.
Authored by
Phillip Thurston
Script to make sure everything is set when converting a repo from old legacy structures to our new unified k8s and serverless structure.
Run with:
bash <(curl -s https://git.netzilla.io/-/snippets/26/raw)
or to clear npm modules, lock and install/build run with:
bash <(curl -s https://git.netzilla.io/-/snippets/26/raw) fresh
unifiedConversionScript.sh 20.16 KiB
#!/bin/bash
# This is a script designed to help getting existing projects to work with our
# new unified starter and deployments.
function wordShouldNotExist {
if [[ $(grep -ir --exclude-dir=node_modules --exclude-dir=.git "$1" ./) ]]; then
echo -e "\e[91mThe word '$1' is present in the project\e[0m"
grep -ir --exclude-dir=node_modules --exclude-dir=.git "$1" ./
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] the word '$1' is not in the project"
fi
}
function wordShouldExist {
if [[ $(grep -ir --exclude-dir=node_modules --exclude-dir=.git "$1" ./) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] the word '$1' is in the project"
else
echo -e "\e[91mThe word '$1' is not present in the project\e[0m"
grep -ir --exclude-dir=node_modules --exclude-dir=.git "$1" ./
export CONFIG_CHECK_FAIL=true
fi
}
# If run with argument "fresh" it will clear npm modules lock and then reinstall and build
if [[ $1 == "fresh" ]]; then
echo -e "Removing node_modules/ dir in current directory"
rm -rf node_modules
echo -e "Removing package-lock.json in current directory"
rm package-lock.json
echo -e "Installing NPM modules and running build"
set -e
npm install
npm run build
set +e
fi
if [ $(grep -oP 'PROD_SUBDOMAIN: \K[a-z\-]+' .gitlab-ci.yml 2>/dev/null) ]; then
PROD_SUBDOMAIN=$(grep -oP 'PROD_SUBDOMAIN: \K[a-z\-]+' .gitlab-ci.yml)
echo -e "prod subdomain is set as \"$PROD_SUBDOMAIN\""
fi
if [ $(grep -oP '^ SERVERLESS_ENV: \K[a-z\w]+' .gitlab-ci.yml 2>/dev/null) ]; then
SERVERLESS_ENV=$(grep -oP '^ SERVERLESS_ENV: \K[a-z\w]+' .gitlab-ci.yml)
echo -e "ENV is set as \"$SERVERLESS_ENV\""
fi
if [[ $(grep LEGACY_DEPLOY .gitlab-ci.yml 2>/dev/null) ]]; then
LEGACY_DEPLOY='true'
echo -e "Legacy Deploy detected"
fi
if [[ $(grep NO_SERVERLESS_ENDPOINT .gitlab-ci.yml 2>/dev/null) ]]; then
NO_SERVERLESS_ENDPOINT='true'
echo -e "Scheduled lambda detected"
fi
if [[ $(grep NPM_PACKAGE_PUBLISH .gitlab-ci.yml 2>/dev/null) ]]; then
NPM_PACKAGE_PUBLISH='true'
echo -e "NPM repo project detected"
fi
CI_PROJECT_URL='https://test.com'
CI_COMMIT_REF_SLUG='testbranch'
if [[ ! -z ${SERVERLESS_ENV+x} ]]; then
if [[ -f "./serverless.yml" ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The serverless.yml exists"
else
echo -e "\e[91mThe serverless Configuration is missing!\e[0m The serverless.yml file is missing. Is this a serverless project? Please check the lambda starter template for examples"
exit 1
fi
if [[ -f "./package.json" ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The package.json exists"
else
echo -e "\e[91mThe serverless Configuration is invalid!\e[0m The package.json file is missing. Is this a node project? Please check the lambda starter template for examples"
exit 1
fi
if [[ ! $(grep 'junit.xml' ./.gitignore) ]]; then
echo -e "\e[91mjunit.xml isn't in the gitignore\e[0m"
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] jest results are in the gitignore"
fi
if [[ -z ${NO_SERVERLESS_ENDPOINT+x} ]]; then
if [ -z ${PROD_SUBDOMAIN+x} ]; then
echo -e "\e[91mThe PROD_SUBDOMAIN variable is unset!\e[0m This script is designed to be run with gitlab CI/CD Jobs"
export CONFIG_CHECK_FAIL=true
fi
if [[ $PROD_SUBDOMAIN == "changeme" ]]; then
echo -e "\e[91mThe Prod subdomain is still set to it's default!\e[0m You must change the project's PROD_SUBDOMAIN variable to a unqiue subdomain in the .gitlab-ci.yml file"
export CONFIG_CHECK_FAIL=true
elif [[ -z ${PROD_SUBDOMAIN+x} ]]; then
echo -e "\e[91mThe test cannot run!\e[0m because the PROD_SUBDOMAIN is not set!"
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The Production subdomain appears to be valid and is: $PROD_SUBDOMAIN"
fi
if [[ $(echo $PROD_SUBDOMAIN | grep --perl-regexp [^a-z0-9\-]) ]]; then
echo -e "\e[91mThe Prod subdomain variable is formatted improperly or like a subzone!\e[0m You must change the PROD_SUBDOMAIN variable in the .gitlab-ci.yml file to be a valid subdomain. Containing only (lowercase) a-z, 0-9, or hyphens (-)"
export CONFIG_CHECK_FAIL=true
elif [[ -z ${PROD_SUBDOMAIN+x} ]]; then
echo -e "\e[91mThe test cannot run!\e[0m because the PROD_SUBDOMAIN is not set!"
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The Production subdomain appears to be RFC compliant"
fi
if [ $(echo $PROD_SUBDOMAIN | grep --perl-regexp "\-?stage\-?") ] || [ $(echo $PROD_SUBDOMAIN | grep --perl-regexp "\-?dev\-?") ]; then
echo -e "\e[91mThe Prod subdomain variable is is confusing\e[0m You must change the PROD_SUBDOMAIN variable in the .gitlab-ci.yml file to not have the words stage or dev in it since our automatically generated staging and dev URLS contain those words and we don't want confusion."
export CONFIG_CHECK_FAIL=true
elif [[ -z ${PROD_SUBDOMAIN+x} ]]; then
echo -e "\e[91mThe test cannot run!\e[0m because the PROD_SUBDOMAIN is not set!"
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The Production subdomain appears to play nice with our automation"
fi
if [[ $(grep 'serverless-domain-manager' ./serverless.yml) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The serverless domain manager plugin seems to be in the config"
else
echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the serverless-domain-manager plugin is installed. Please check the lambda starter template for examples"
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep "production: \${file(\.\/\.gitlab-ci\.yml):variables\.PROD_SUBDOMAIN}\.\${env:CERTIFICATE_DOMAIN, 'localhost'}" ./serverless.yml) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The production URI is properly referenced in the config"
else
echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the production domain variable is properly formatted in the serverless.yml file. Please check the lambda starter template for examples"
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep "\${file(\.\/\.gitlab-ci\.yml):variables\.PROD_SUBDOMAIN}-stage0\.\${env:CERTIFICATE_DOMAIN, 'localhost'}" ./serverless.yml) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The staging URI is properly referenced in the config"
else
echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the staging domain variable is properly formatted in the serverless.yml file. Please check the lambda starter template for examples"
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep 'serverless-domain-manager' ./package.json) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] Serverless Domain Manager seems to be installed in the package.json"
else
echo -e "\e[91mThere are missing npm packages!\e[0m Serverless Domain Manager doesn't seem to be installed. To do so simply run 'npm install serverless-domain-manager --save-dev'. This results in having automatic domains in serverless"
export CONFIG_CHECK_FAIL=true
fi
fi
if [[ $(grep 'serverless-domain-manager' ./serverless.yml) && ! -z ${NO_SERVERLESS_ENDPOINT+x} ]]; then
echo -e "\e[91mThe serverless Configuration out of order!\e[0m It seems that domains are configured for this project even though it is listed as having no active endpoints. If this project is going to be unreachable to the outside world please cleanup the serverless.yml and package.json. If any part of the project will be accessed via a URI then make sure the NO_SERVERLESS_ENDPOINT variable is unset."
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] There don't seem to be any conflicting endpoint configurations."
fi
if [[ -z ${LEGACY_DEPLOY+x} ]]; then
if [[ $(grep "service: \${self:custom.env}*" serverless.yml) && -z ${LEGACY_DEPLOY+x} ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The env opt is part of the service name"
if [[ $(grep "service:*" serverless.yml | sed -r 's/service:\s\$\{self:custom\.env\}(.*$)/\1/' | wc -c) -le 18 ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The service name isn't too long."
else
echo -e "\e[91mThe deployment will fail!\e[0m AWS (specifically AmazonIdentityManagement) requires that all names are shorter than 64 characters. With how our selected variables and settings are configured our names can't be longer than 18 characters AFTER the \${self:custom.env}.\nThe service name computed ( $(grep "service:*" serverless.yml | sed -r 's/service:\s\$\{self:custom\.env\}(.*$)/\1/') ) counted out is $(grep "service:*" serverless.yml | sed -r 's/service:\s\$\{self:custom\.env\}(.*$)/\1/' | wc -c) characters long. Make sure it's 18 or less characters"
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep "service: \${self:custom.env}-changeme" serverless.yml) ]]; then
echo -e "\e[91mThe service name is still set to it's default!\e[0m You must change the project's service name to a unqiue name in the serverless.yml file. The default is \"\${self:custom.env}-changeme\" and it should be changed to something like \"\${self:custom.env}-newproject\""
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The service name is not the default."
fi
else
echo -e "\e[91mThe deployment is not future proof!\e[0m all deployments must have unique names so if we ever add any additional env's or deployments for this project we need some way to make sure the service name is unique. Make sure you have something like \"service: \${self:custom.env}-example\" listed in the serverless.yml even if we never plan to make this a multi-env project. We want to stay consistent. Feel free to check out our starter for more examples."
export CONFIG_CHECK_FAIL=true
fi
fi
if [[ $(grep 'deploymentBucket' ./serverless.yml) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The deployment bucket seems to be referenced in the config"
else
echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the the deploymentBucket is set to our centralized s3 bucket. Please check the lambda starter template for examples"
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep 'logRetentionInDays' ./serverless.yml) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] Log retention seems to be referenced in the config"
else
echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure the the logRetentionInDays is set to a sane value. Please check the lambda starter template for examples"
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep --perl-regexp '[\#[:space:]]+profile' ./serverless.yml) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] AWS creds will use the default profile"
else
echo -e "\e[91mThe serverless Configuration is invalid!\e[0m Whoops, our CI uses the default AWS profile. Make sure you comment out the 'provider.profile' section of the serverless.yml. If you're unsure how to do this reference the lambda starter."
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep 'service: serverless-example' ./serverless.yml) ]]; then
echo -e "\e[91mThe service name is still set to it's default!\e[0m You must make sure the 'service' parameter of the serverless.yml is not set to its default. Please change it to something unique to all our other projects."
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The serverless service name has been changed from the default."
fi
if [ -z ${COMPANY_NAME+x} ] && [ -z ${SERVERLESS_ENV+x} ]; then
echo -e "\e[91mNo companies/envs are defined!\e[0m Our automation either looks for a list of companies to use (SERVERLESS_ENV) or a single company to use (SERVERLESS_ENV). If the projet is multi-tenanted with multiple ENVS then use the SERVERLESS_ENV other wise for a deployment only needed by a single company use COMPANY_NAME. One of these two variables must be defined."
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] A company seems to have been configured."
fi
if [ ! -z ${COMPANY_NAME+x} ] && [ ! -z ${SERVERLESS_ENV+x} ]; then
echo -e "\e[91mThe COMPANY_NAME variable and SERVERLESS_ENV are both defined!\e[0m Our automation either looks for a list of companies to use (SERVERLESS_ENV) or a single company to use (SERVERLESS_ENV). If the projet is multi-tenanted with multiple ENVS then use the SERVERLESS_ENV other wise for a deployment only needed by a single company use COMPANY_NAME. Do not use both variables at the same time."
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] Only one list of companies is provided."
fi
if [[ ! -z ${SERVERLESS_ENV+x} ]]; then
TEST_ENV=( $SERVERLESS_ENV )
CONTROL_ENV=( blockchainjedi blockchainjedi connect connect codex codex green green blue blue give give ) # Gross https://stackoverflow.com/a/28161520/9879357
INVALID_ENV=(`echo ${TEST_ENV[@]} ${CONTROL_ENV[@]} | tr ' ' '\n' | sort | uniq -u `)
if [ ! -z ${INVALID_ENV+x} ]; then
echo -e "\e[91mThe SERVERLESS_ENV variable has an unknown element!\e[0m You may be adding a brand new type of ENV. that's fine, just have DevOps quicly add it to the automation or use the single target deployment seen in the lambda starter repo. The accepted values for the SERVERLESS_ENV varible are\n=====\n- blockchainjedi \n- connect \n- codex \n- green \n- blue \n- give\n====="
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The listed server enviroments are all supported"
fi
if [[ $(grep 'CI_PROJECT_URL' ./serverless.yml) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The serverless functions seem to have their descriptions"
else
echo -e "\e[91mThe serverless Configuration is invalid!\e[0m You must make sure each function has a description that contains our \${env:CI_PROJECT_URL}. This is important for management for dynamically deployed functions. Please check the lambda starter template for examples"
export CONFIG_CHECK_FAIL=true
fi
if [[ $SERVERLESS_ENV =~ [A-Z] ]]; then
echo -e "\e[91mThe SERVERLESS_ENV variable should have no capitalization!\e[0m make sure that your listed ENVS are all lowercase."
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The listed server enviroments are all lowercase"
fi
fi
if [[ $(grep '\"test\": \"jest --watch\"' ./package.json) || $(grep '\"test\": \"jest --watchAll\"' ./package.json) ]]; then
echo -e "[\e[91mERROR\e[0m] Jest is set to 'watch' this means that the job will never complete. Make sure you have 'jest --coverage' somewhere in the 'test' script of your package.json."
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep 'jest-junit' ./package.json) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] JUnit reports will be used and reported to the git system for easy analysis."
else
echo -e "[\e[93mWARNING\e[0m] JUnit reporting doesn't seem to be enabled. To do so simply run 'npm install jest-junit --save-dev' and add jest-junit to the reporters array in the jest config. this results in being able to see test results directly in merge requests."
fi
if [[ $(grep 'cobertura' ./jest.config.js) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] Cobertura reports should be generated to help the git system show untested lines in git diffs."
else
echo -e "[\e[93mWARNING\e[0m] Cobertura reporting doesn't seem to be enabled. To do so simply add 'cobertura' to the coverageReporters array in the jest config."
fi
if [[ ! $(grep '\"test\": \"jest --coverage\"' ./package.json) ]]; then
echo -e "[\e[93mWARNING\e[0m] Jest didn't output any proper coverage reporting. Make sure you have 'jest --coverage' somewhere in the 'test' script of your package.json. From that point on you should see code coverage reports with pipelines in the git system"
fi
if [[ $(grep 'deploy:prod' ./package.json) ]]; then
echo -e "[\e[93mWARNING\e[0m] deploy scripts are still in the package.json"
fi
if [[ ! -z ${LEGACY_DEPLOY+x} && ! $(grep -i 'warning' ./.gitlab-ci.yml) ]]; then
echo -e "[\e[93mWARNING\e[0m] Legacy warning message is missing"
fi
fi
if [[ $(grep -oP 'COMPANIES: \K[a-z\-]+' .gitlab-ci.yml 2>/dev/null) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] The COMPANIES env variable is declared"
else
echo -e "[\e[91mERROR\e[0m] The COMPANIES env variable is unset. You must have that variable declared."
CONFIG_CHECK_FAIL=true
fi
if [[ -f package.json ]]; then
if [[ -z ${NPM_PACKAGE_PUBLISH+x} ]]; then
if [[ -f .npmrc ]]; then
echo -e "[\e[92mSUCCESS\e[0m] the .npmrc file exists."
if [[ $(grep '\@blockbrothers:registry=https:\/\/npm\.connectblockchain\.net' .npmrc) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] the BB NPM registry is in the .npmrc file"
else
echo -e "\e[91mThe BB NPM registry isn't properly listed in the project's .npmrc file!\e[0m You must make sure the .npmrc file exists and has the bb registry inside of it."
export CONFIG_CHECK_FAIL=true
fi
else
echo -e "\e[91mThe .npmrc file is missing!\e[0m You must make sure the .npmrc file exists and has the bb registry inside of it."
export CONFIG_CHECK_FAIL=true
fi
fi
if [[ -x .husky/pre-commit ]]; then
echo -e "[\e[92mSUCCESS\e[0m] the the husky version bump script is present and executable"
else
echo -e "[\e[91mError\e[0m] The husky version bump script is either not present or executable"
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep '\"prepublish\"' package.json) ]]; then
echo -e "\e[91mprepublish is used in in the package.json file!\e[0m using the prepublish script in NPM scripts is not advisable since it will run those commands on any install command including on prod servers. Instead replace it with prepublishOnly which will only run those commands before you publish your NPM script and not on every install."
export CONFIG_CHECK_FAIL=true
fi
fi
if [[ ! $(grep -i 'Phillip Thurston' Dockerfile) ]]; then
echo -e "[\e[91mError\e[0m] It looks like the dockerfile hasn't been setup yet."
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] the Dockerfile has been setup"
fi
if [[ ! $(grep -i '\"dockerport\": 3000' package.json) ]]; then
echo -e "[\e[91mError\e[0m] It looks like the dockerport config isn't in the package.json."
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The dockerport is configured"
fi
if [[ ! $(grep -i 'npm_package_config_dockerport:3000' package.json) ]]; then
echo -e "[\e[91mError\e[0m] It looks like the Docker run command isn't using a good port."
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] Docker run is using the correct ports"
fi
if [[ ! $(grep -i 'EXPOSE 3000' Dockerfile) ]]; then
echo -e "[\e[91mError\e[0m] Correct port is set in the dockerfile"
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The dockerfile is either not using a port or using an invalid port"
fi
if [[ ! $(grep -i '\"husky install\"' package.json) ]]; then
echo -e "[\e[91mError\e[0m] the version bump script hook isn't in the package.json"
export CONFIG_CHECK_FAIL=true
else
echo -e "[\e[92mSUCCESS\e[0m] The version bump script git hook is active."
fi
if [[ -f .travis.yml ]]; then
echo -e "[\e[91mError\e[0m] There is a travis file in this project. We don't use travis"
export CONFIG_CHECK_FAIL=true
fi
if [[ -f .dockerignore ]]; then
echo -e "[\e[92mSUCCESS\e[0m] .dockerignore is present"
else
echo -e "[\e[91mError\e[0m] The .dockerignore file is not present"
export CONFIG_CHECK_FAIL=true
fi
if [[ $(grep -ir --exclude-dir=node_modules --exclude-dir=.git "/health" ./) ]]; then
echo -e "[\e[92mSUCCESS\e[0m] There seems to be a healthcheck"
else
echo -e "[\e[91mError\e[0m] There doesn't seem to be a healthcheck api endpoint. make sure we have a /health endpoint that at minimum responds with a 200 GET request"
export CONFIG_CHECK_FAIL=true
fi
wordShouldNotExist starter
wordShouldNotExist image:build
wordShouldNotExist loyalty # Keyword check in package.json
wordShouldExist MONGODB_URI_ # Make sure we are using the new mongo FORMAT
if [ ! -z ${CONFIG_CHECK_FAIL+x} ]; then
echo -e "Quick tips:\ngrep -ir --exclude-dir=node_modules --exclude-dir=.git \"<i>\" ./"
exit 1
fi
echo -e "\e[91mRemove the Fork relationship in the project\e[0m"
echo -e "\e[91mSet MR to fail if pipeline isn't successful\e[0m"
echo -e "\e[91mSet tags to protected to prevent prod pushes\e[0m"
Please register or sign in to comment