Technical Articles
SAP Data Hub GitHub integration
As of version 2.7.3, there is no out of the box GitHub integration build into SAP Data Hub yet. However, with the out of the box Read File and Write File Operators, there is full access to all SAP Data Hub artefacts, so that based on the also out of the box Python3 Operator in conjunction with the PyGithub project, it is not difficult to store SAP Data Hub Repository Objects in GitHub and retrieve them back from there.
To demonstrate this, I wrote both a GitWrite and a GitRead Operator including a respective Docker File as well as demo Graphs how to use them (see appendix). The Sleep Operator is there for convenience only, to make it easier to get to the Wiretaps.
As a precondition, you need either your GitHub user name and password, or better a personal access token or in case of an enterprise account both your base URL and access token:
Both Operators accept an SAP Data Hub Repository directory and store it into or retrieve it frrom GitHub:
Then in GitHub, you get your version control respectively:
Both Operators loop through the directory provided while logging their activities to the debug and signalling completion to the stop Output Port:
Appendix
Docker File
FROM $com.sap.sles.base
RUN pip3.6 install PyGithubGitWrite Script
from github import Github
import re
# First create a Github instance:
# using username and password
# g = Github("user", "password")
# or using an access token
g = Github(api.config.access_token)
# Github Enterprise with custom hostname
# g = Github(base_url="https://{hostname}/api/v3", login_or_token="access_token")
def on_input(in_filename, in_file):
# Then play with your Github objects:
filename = re.sub(r"^/[^/]*/", "", in_filename.body)
file = in_file.body
end_of_sequence = in_file.attributes.get("storage.endOfSequence")
repo = g.get_user().get_repo(api.config.repo)
try:
contents = repo.get_contents(filename)
repo.update_file(contents.path, api.config.comment, file, contents.sha, branch=api.config.branch)
api.send("debug", "Updated " + filename)
except:
repo.create_file(filename, "Initial commit", file, branch=api.config.branch)
api.send("debug", "Created " + filename)
if end_of_sequence:
api.send("stop", end_of_sequence)
api.set_port_callback(["inFilename", "inFile"], on_input)GitRead Script
from github import Github
import re
# First create a Github instance:
# using username and password
# g = Github("user", "password")
# or using an access token
g = Github(api.config.access_token)
# Github Enterprise with custom hostname
# g = Github(base_url="https://{hostname}/api/v3", login_or_token="access_token")
def on_input(in_path):
# Then play with your Github objects:
repo = g.get_user().get_repo(api.config.repo)
contents = repo.get_contents(in_path)
while contents:
file_content = contents.pop(0)
headers = {"storage.path": re.sub(r"^/[^/]*/", "", file_content.path)}
if file_content.type == "dir":
contents.extend(repo.get_contents(file_content.path))
else:
api.send("output", api.Message(file_content.decoded_content, headers))
api.send("debug", "Read " + file_content.path)
api.send("stop", contents)
api.set_port_callback("inPath", on_input)Sleep Script
$.setPortCallback("input",onInput);
function onInput(ctx,s) {
var now = new Date().getTime();
while (new Date().getTime() < now + $.config.getInt("duration")) {
}
$.output(null);
}de.architectsap.github.write
{
"properties": {},
"description": "",
"processes": {
"wiretap1": {
"component": "com.sap.util.wiretap",
"metadata": {
"label": "Wiretap",
"x": 539.9999990463257,
"y": 12,
"height": 80,
"width": 120,
"ui": "dynpath",
"config": {}
}
},
"readfile1": {
"component": "com.sap.storage.read",
"metadata": {
"label": "Read File",
"x": 186,
"y": 72,
"height": 80,
"width": 120,
"config": {
"path": "",
"recursive": true
}
}
},
"graphterminator1": {
"component": "com.sap.util.graphTerminator",
"metadata": {
"label": "Graph Terminator",
"x": 724.9999980926514,
"y": 72,
"height": 80,
"width": 120,
"config": {}
}
},
"sleep1": {
"component": "de.architectsap.sleep",
"metadata": {
"label": "Sleep",
"x": 539.9999990463257,
"y": 132,
"height": 80,
"width": 120,
"extensible": true,
"config": {
"duration": 50000
}
}
},
"constantgenerator1": {
"component": "com.sap.util.constantGenerator",
"metadata": {
"label": "Constant Generator",
"x": 17,
"y": 72,
"height": 80,
"width": 120,
"extensible": true,
"config": {
"content": "/vrep/vflow/subengines/com/sap/python36/operators/de/architectsap"
}
}
},
"gitwrite1": {
"component": "de.architectsap.gitwrite",
"metadata": {
"label": "GitWrite",
"x": 355,
"y": 72,
"height": 80,
"width": 120,
"extensible": true,
"config": {
"access_token": "9947fbedd94dcc11a62df4302a7df69476a65506"
}
}
}
},
"groups": [],
"connections": [
{
"metadata": {
"points": "663.9999990463257,172 691.9999985694885,172 691.9999985694885,112 719.9999980926514,112"
},
"src": {
"port": "output",
"process": "sleep1"
},
"tgt": {
"port": "stop",
"process": "graphterminator1"
}
},
{
"metadata": {
"points": "141,112 181,112"
},
"src": {
"port": "out",
"process": "constantgenerator1"
},
"tgt": {
"port": "inPath",
"process": "readfile1"
}
},
{
"metadata": {
"points": "310,103 350,103"
},
"src": {
"port": "outFilename",
"process": "readfile1"
},
"tgt": {
"port": "inFilename",
"process": "gitwrite1"
}
},
{
"metadata": {
"points": "310,121 350,121"
},
"src": {
"port": "outFile",
"process": "readfile1"
},
"tgt": {
"port": "inFile",
"process": "gitwrite1"
}
},
{
"metadata": {
"points": "479,121 506.99999952316284,121 506.99999952316284,172 534.9999990463257,172"
},
"src": {
"port": "stop",
"process": "gitwrite1"
},
"tgt": {
"port": "input",
"process": "sleep1"
}
},
{
"metadata": {
"points": "479,103 506.99999952316284,103 506.99999952316284,52 534.9999990463257,52"
},
"src": {
"port": "debug",
"process": "gitwrite1"
},
"tgt": {
"port": "in",
"process": "wiretap1"
}
}
],
"inports": {},
"outports": {}
}de.architectsap.guthub.read
{
"properties": {},
"description": "",
"processes": {
"wiretap1": {
"component": "com.sap.util.wiretap",
"metadata": {
"label": "Wiretap",
"x": 370.9999990463257,
"y": 12,
"height": 80,
"width": 120,
"ui": "dynpath",
"config": {}
}
},
"sleep1": {
"component": "de.architectsap.sleep",
"metadata": {
"label": "Sleep",
"x": 370.9999990463257,
"y": 252,
"height": 80,
"width": 120,
"extensible": true,
"config": {
"duration": 50000
}
}
},
"graphterminator1": {
"component": "com.sap.util.graphTerminator",
"metadata": {
"label": "Graph Terminator",
"x": 555.9999980926514,
"y": 192,
"height": 80,
"width": 120,
"config": {}
}
},
"constantgenerator1": {
"component": "com.sap.util.constantGenerator",
"metadata": {
"label": "Constant Generator",
"x": 17,
"y": 132,
"height": 80,
"width": 120,
"extensible": true,
"config": {
"content": "/vflow/subengines/com/sap/python36/operators/de/architectsap"
}
}
},
"writefile1": {
"component": "com.sap.storage.write",
"metadata": {
"label": "Write File",
"x": 370.9999990463257,
"y": 132,
"height": 80,
"width": 120,
"config": {
"mode": "overwrite",
"path": "/vrep/\\${storage.path}"
}
}
},
"gitread1": {
"component": "de.architectsap.gitread",
"metadata": {
"label": "GitRead",
"x": 186,
"y": 132,
"height": 80,
"width": 120,
"extensible": true,
"config": {
"access_token": "9947fbedd94dcc11a62df4302a7df69476a65506"
}
}
},
"wiretap2": {
"component": "com.sap.util.wiretap",
"metadata": {
"label": "Wiretap",
"x": 555.9999980926514,
"y": 72,
"height": 80,
"width": 120,
"ui": "dynpath",
"config": {}
}
}
},
"groups": [],
"connections": [
{
"metadata": {
"points": "494.9999990463257,292 522.9999985694885,292 522.9999985694885,232 550.9999980926514,232"
},
"src": {
"port": "output",
"process": "sleep1"
},
"tgt": {
"port": "stop",
"process": "graphterminator1"
}
},
{
"metadata": {
"points": "141,172 181,172"
},
"src": {
"port": "out",
"process": "constantgenerator1"
},
"tgt": {
"port": "inPath",
"process": "gitread1"
}
},
{
"metadata": {
"points": "310,154 337.99999952316284,154 337.99999952316284,52 365.9999990463257,52"
},
"src": {
"port": "debug",
"process": "gitread1"
},
"tgt": {
"port": "in",
"process": "wiretap1"
}
},
{
"metadata": {
"points": "310,172 365.9999990463257,172"
},
"src": {
"port": "output",
"process": "gitread1"
},
"tgt": {
"port": "inFile",
"process": "writefile1"
}
},
{
"metadata": {
"points": "310,190 337.99999952316284,190 337.99999952316284,292 365.9999990463257,292"
},
"src": {
"port": "stop",
"process": "gitread1"
},
"tgt": {
"port": "input",
"process": "sleep1"
}
},
{
"metadata": {
"points": "494.9999990463257,172 522.9999985694885,172 522.9999985694885,112 550.9999980926514,112"
},
"src": {
"port": "outFilename",
"process": "writefile1"
},
"tgt": {
"port": "in",
"process": "wiretap2"
}
}
],
"inports": {},
"outports": {}
}
Be the first to leave a comment
You must be Logged on to comment or reply to a post.