This blog post is part of a series on SAP Datasphere and SAP HANA Cloud CI/CD. Before reviewing the details in this blog post, I recommend checking "SAP Datasphere SAP HANA Cloud HDI CI/CD Automation Approach" for an overview of the use case scenario, toolset and concepts.
cds add pipeline
@Library('piper-lib-os') _
node() {
stage('prepare') {
deleteDir()
checkout scm
setupCommonPipelineEnvironment script: this,
verbose: true
}
stage('build') {
mtaBuild script: this,
mtaBuildTool: 'cloudMbt',
verbose: true
}
stage('deploy') {
cloudFoundryDeploy script: this,
deployTool:'mtaDeployPlugin',
verbose: true
}
stage('Validation') {
npmExecuteScripts script: this,
verbose: true
}
stage('tmsUpload') {
tmsUpload script: this
}
stage('Trigger_DWC_Pipeline') {
build 'HDACSM/dwc_cli_ctms/master'
}
}
steps:
### Stage Build
mtaBuild:
buildTarget: 'CF'
### Stage Deploy CF deploy
cloudFoundryDeploy:
deployTool: 'mtaDeployPlugin'
deployType: 'standard'
cloudFoundry:
org: 'CF-ORG-ABC-E2E-DEMOS'
space: 'ABC-E2E-DEMOS-DWC-SPACE'
credentialsId: 'CF-CREDENTIALSID'
database_id: XeeabcdX-abcd-481d-abcd-00b0417Xabcd
### Stage Validation,
#Execute npm script 'test' to validate db artifacts.
npmExecuteScripts:
buildDescriptorExcludeList:
- db/package.json
runScripts:
- "test"
### Stage tmsUpload, Trigger cTMS to move through landscape
tmsUpload:
credentialsId: 'BTP-TMS'
nodeName: 'IMPORT_DEV'
verbose: 'true'
ID: MVP_SP
version: 0.0.1
modules:
# --------------------Deployer (side car)--------------
- name: db # deployer
# -----------------------------------------------------
type: hdb
path: db
parameters:
app-name: HDIDEV
requires:
- name: HDI_SPDEV # Depends on the HDI Container
properties:
TARGET_CONTAINER: ~{hdi-container-name}
- name: DWC.UPS_SQL_SP_PROJECTDEV
group: SERVICE_REPLACEMENTS
properties:
key: ups_schema_access
service: '~{ups_sql_sp}'
resources:
# --------------------HDI Container--------------
- name: HDI_SPDEV
# -----------------------------------------------
type: com.sap.xs.hdi-container
parameters:
config:
database_id: Xee99a70-abcd-481d-abcd-00b0417Xabcd # to deploy against DWC HC
schema: SP_PROJECTDEV_DWC_HDI
service-name: SP_PROJECTDEV_DWC_S1
properties:
hdi-container-name: ${service-name}
# --------------------UPS user provided service --------------
# to be created first in the CF space in which this HDI shared service gets created.
# Credential you get from DWC Space management. You use this service to access DWC views
- name: DWC.UPS_SQL_SP_PROJECTDEV
# # --------------------------------------------------------------
type: org.cloudfoundry.existing-service
parameters:
service-name: UPS_SQL_SP_PROJECTDEV
properties:
ups_sql_sp: ${service-name}
_schema-version: "3.1"
ID: MVP_SP.config.first
extends: MVP_SP
modules:
- name: db
type: hdb
path: db
parameters:
app-name: HDIQA
resources:
- name: VI_HDI_SPDEV
type: com.sap.xs.hdi-container
parameters:
service: hana
service-plan: hdi-shared
config:
database_id: Xee99a70-abcd-481d-abcd-00b0417Xabcd
schema: SP_PROJECT_QA_DWC_HDI
service-name: SP_PROJECTQA_DWC_S1
properties:
hdi-container-name: ${service-name}
- name: DWC.UPS_SQL_SP_PROJECTDEV
type: org.cloudfoundry.existing-service
parameters:
service-name: UPS_SQL_SP_PROJECTQA
@Library('piper-lib-os') _
node() {
stage('prepare') {
deleteDir()
checkout scm
setupCommonPipelineEnvironment script: this
verbose: true
}
stage('build') {
withCredentials([
usernamePassword(credentialsId: "DWC_CredentialsID",
usernameVariable: 'DWC_USER',
passwordVariable: 'DWC_PASS')
])
{
dockerExecute(
script: this,
dockerImage: 'vishwagi/puppeteer-dwc-node-docker:latest',
dockerEnvVars: ['DWC_PASS':'$DWC_PASS','DWC_USER':'$DWC_USER',])
{
sh 'node Build.js';
}
verbose: true
}
}
stage('deploy') {
withCredentials([
usernamePassword(credentialsId: "DWC_CredentialsID",
usernameVariable: 'DWC_USER',
passwordVariable: 'DWC_PASS')
])
{
dockerExecute(
script: this,
dockerImage: 'vishwagi/puppeteer-dwc-node-docker:latest',
dockerEnvVars: ['DWC_PASS':'$DWC_PASS','DWC_USER':'$DWC_USER',])
{
sh 'node Deploy.js';
}
verbose: true
}
}
stage('Validation') {
npmExecuteScripts script: this,
verbose: true
}
}
steps:
### Stage Build and Deploy set env variables
dockerExecute:
dockerEnvVars:
DWC_URL: 'https://dwc-ab-abcd.eu10.hcs.cloud.sap/'
DWC_PASSCODE_URL: 'https://dwc-ab-abcd.authentication.eu10.hana.ondemand.com/passcode'
HDIDEV: 'SP_PROJECTDEV_DWC_HDI'
HDIQA: 'SP_PROJECT_QA_DWC_HDI'
SPACE: 'SP_PROJECTDEV'
SPACEQA: 'SP_PROJECTQA'
LABELQA: 'DWC_QA'
ENTITIES: ''
SPACE_DEFINITION_FILE: 'SP_PROJECTDEV.json'
NEW_SPACE_DEFINITION_FILE: 'SP_PROJECTQA.json'
### Stage Validation, Execute npm script 'test' to validate db artifacts.
npmExecuteScripts:
buildDescriptorList:
- srv/package.json
runScripts:
- "test"
Dockerfile.
FROM geekykaran/headless-chrome-node-docker:latest
LABEL version="1.0"
LABEL author = "Vishwa Gopalkrishna"
RUN apt update; \
apt upgrade;
RUN npm cache clean -f; \
npm install n -g; \
n stable;
ADD package.json package-lock.json /
# The steps below are to enhance Docker image
# otherwise the image from Docker Hub can be used as is.
# open terminal in the same folder as Dockerfile and run below
# Command #1 to create package.json file.
# npm init --yes
# Command #2 install dependencies, these would be written in package.json file
# npm install @sap/dwc-cli fs-extra puppeteer path
# Now if you check the package.json and package-lock.json you should see the dependency list.
RUN npm install
# #3 Build command
# docker build -t vishwagi/puppeteer-dwc-node-docker:latest .
# Version 1.0 image has below packages
# ***IMPORTANT other @sap/dwc-cli version may need changes to Build.js
# "@sap/dwc-cli": "^2022.14.0",
# "fs-extra": "^10.1.0",
# "path": "^0.12.7",
# "puppeteer": "^15.3.0"
const puppeteer = require("puppeteer");
const exec = require("child_process").exec;
const fs = require('fs-extra');
const SPACE_DEFINITION_FILE = process.env.SPACE_DEFINITION_FILE;
const NEW_SPACE_DEFINITION_FILE = process.env.NEW_SPACE_DEFINITION_FILE;
const SPACE = process.env.SPACE;
const SPACEQA = process.env.SPACEQA;
const LABELQA = process.env.LABELQA;
const ENTITIES = process.env.ENTITIES;
const HDIDEV = process.env.HDIDEV;
const HDIQA = process.env.HDIQA;
const DWC_URL = process.env.DWC_URL;
const DWC_PASSCODE_URL = process.env.DWC_PASSCODE_URL;
const USERNAME = process.env.DWC_USER;
const PASSWORD = process.env.DWC_PASS;
let page;
const getPasscode = async () => {
console.log('Inside get passcode module');
await page.waitForSelector('div.island > h1 + h2', {visible: true, timeout: 5000});
await page.reload();
return await page.$eval('h2', el => el.textContent);
}
const execCommand = async (command) => new Promise(async (res, rej) => {
const passcode = await getPasscode();
console.log('Passcode OK');
const cmd = `${command} -H ${DWC_URL} -p ${passcode}`;
console.log('command for space download', cmd);
exec(cmd, (error, stdout, stderr) => {
if (error) {
console.error(`error: ${error.message}`);
if (error.code === 1) {
res({ error, stdout, stderr });
}else {
rej({ error, stdout, stderr });
}
}
else{
res({ error, stdout, stderr });
}
console.log(`stdout:\n${stdout}`);
console.log(`error:\n${error}`);
console.log(`stderr:\n${stderr}`);
});
});
(async () => {
const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox']});
page = await browser.newPage();
await page.goto(DWC_PASSCODE_URL);
await page.waitForSelector('#logOnForm', {visible: true, timeout: 5000});
if (await page.$('#logOnForm') !== null) {
await page.type('#j_username', USERNAME);
await page.type('#j_password', PASSWORD);
await page.click('#logOnFormSubmit');
}
//--------- READ DEV SPACE ------------------//
console.log(process.env);
await execCommand(`dwc cache-init`);
await execCommand(`dwc spaces read -s ${SPACE} -o ${SPACE_DEFINITION_FILE} -d ${ENTITIES}`);
//--------- CREATE/UPDATE QA SPACE ------------------//
const spaceContent = await fs.readFile(SPACE_DEFINITION_FILE, 'utf-8')
console.log('Read file');
const replacer = new RegExp(HDIDEV, 'gi')
const spaceContentQA = spaceContent.replace(replacer, HDIQA);
// parse the downloaded space definition file
const spaceDefinition = JSON.parse(spaceContentQA);
// We need to update the SPACE ID as well the dbuser as it is specific to space
// First lets get the current space name and label and get the dbusername.
const dbuser_name = SPACE +'#'+ spaceDefinition[SPACE].spaceDefinition.label;
// copy the dbuser details into a placeholder for now, we will attach the same config to new dbuser.
const dbuser_details = spaceDefinition[SPACE].spaceDefinition.dbusers[dbuser_name];
console.log(dbuser_details);
console.log(spaceDefinition[SPACE].spaceDefinition.dbusers)
// update to new dbusername
const dbuser_name_new = SPACEQA+'#'+LABELQA;
// const dbuserjson = JSON.stringify([dbuser_name_new]: dbuser_details)
// parse the created json otherwise it would add double escape / later
const dbuser_json = JSON.parse(JSON.stringify({ [dbuser_name_new] : dbuser_details}));
// Udpate laberl and dbuser details with new one
spaceDefinition[SPACE].spaceDefinition.label = LABELQA;
spaceDefinition[SPACE].spaceDefinition.dbusers = dbuser_json;
// Change root node to new QA space
var json = JSON.stringify({ [SPACEQA] : spaceDefinition[SPACE]});
// console.log(json);
// Write the space details to the file to be consumed by deploy later.
await fs.writeFile(NEW_SPACE_DEFINITION_FILE, json, 'utf-8');
console.log('MAIN after executing commands');
await browser.close();
})();
const puppeteer = require("puppeteer");
const path = require('path');
const exec = require("child_process").exec;
const NEW_SPACE_DEFINITION_FILE = process.env.NEW_SPACE_DEFINITION_FILE;
const DWC_URL = process.env.DWC_URL;
const DWC_PASSCODE_URL = process.env.DWC_PASSCODE_URL;
const USERNAME = process.env.DWC_USER;
const PASSWORD = process.env.DWC_PASS;
let page;
const getPasscode = async () => {
console.log('Inside get passcode module');
await page.waitForSelector('div.island > h1 + h2', {visible: true, timeout: 20000});
await page.reload();
return await page.$eval('h2', el => el.textContent);
}
const execCommand = async (command) => new Promise(async (res, rej) => {
const passcode = await getPasscode();
console.log('Passcode OK');
const cmd = `${command} -H ${DWC_URL} -p ${passcode}`;
console.log('command for space download', cmd);
exec(cmd, (error, stdout, stderr) => {
if (error) {
console.error(`error: ${error.message}`);
if (error.code === 1) {
res({ error, stdout, stderr });
}else {
rej({ error, stdout, stderr });
}
}
else{
res({ error, stdout, stderr });
}
console.log(`stdout:\n${stdout}`);
console.log(`error:\n${error}`);
console.log(`stderr:\n${stderr}`);
});
});
(async () => {
const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox']});
page = await browser.newPage();
await page.goto(DWC_PASSCODE_URL);
await page.waitForSelector('#logOnForm', {visible: true, timeout: 10000});
if (await page.$('#logOnForm') !== null) {
await page.type('#j_username', USERNAME);
await page.type('#j_password', PASSWORD);
await page.click('#logOnFormSubmit');
}
// console.log(process.env);
await execCommand(`dwc cache-init`);
//--------- CREATE SPACE ------------------//
// The below command will create dwc space from the supplied .json(-f) file
await execCommand(`dwc spaces create -f ${NEW_SPACE_DEFINITION_FILE}`);
console.log('MAIN after executing commands');
await browser.close();
})();
I'll add a video here of the code walkthrough and end-to-end demo soon; watch this space.
You must be a registered user to add a comment. If you've already registered, sign in. Otherwise, register and sign in.
User | Count |
---|---|
16 | |
11 | |
8 | |
8 | |
6 | |
6 | |
6 | |
5 | |
5 | |
5 |