mirror of https://github.com/kubeflow/examples.git
Merge dd6c9d4aa7 into f43e99ff22
This commit is contained in:
sefgsefg
GitHub
commit
2ccd411e5e
File diff suppressed because it is too large
Load Diff
|
|
@ -1,68 +1,68 @@
|
|||
# Chinese-multiperson-voice-recognition-using-transfer-learning
|
||||
This is an example of applying transfer learning to the Chinese multi-person voice recognition application. Transfer learning is an AI technique used to enhance the training accuracy of use cases when the dataset is small or the training accuracy is low given the high noise of the original dataset. Multi-person voice recognition is known to contain high noise in the dataset. Chinese voice voice recognition has gained much progress recently thanks to the effort by the big name company such as Google. However many issues remain unsolved. Multi-person Chinese voice recognition is one of them. This example provieds not only multi-person Chinese voice sample dataset, but applied a transfer learning technique to the CNN trained model of the Chinese voice samples dataset. Satisfactory results can be achieved through transfer learning after an initial CNN training.
|
||||
This example provides a feasibility evidence of the transfer learning techniques, and it is our wish to convert the transfer learning technique to a Kubeflow asset through this illustration case. A transfer learning pipeline will be constructed to make kubeflow user easy to adapt to their model for training accuracy enhancement. Eventually, other users can benefit from such convenient features of the kubeflow resources.
|
||||
|
||||
usage briefing:
|
||||
1.Process audio files and convert them into spectrograms.
|
||||
2.Establish experimental data, divide them into 3 categories, and set them into CNN network training.
|
||||
3.Perform two training sessions to improve accuracy.
|
||||
4.Compare training methods.
|
||||
|
||||
Tools used:
|
||||
1. TensorFlow
|
||||
2. Anaconda
|
||||
3. Python3.7
|
||||
|
||||
1. preprocess(spectrograms production)
|
||||
|
||||
|
||||
|
||||
2. import spectrogram files.
|
||||
|
||||
|
||||
|
||||
|
||||
3. build training dataset:
|
||||
divide the dataset into training, validation, and testing sets.
|
||||
|
||||
|
||||
|
||||
4. build CNN taining:
|
||||
|
||||
|
||||
|
||||
5. first training
|
||||
|
||||
|
||||
|
||||
6. first training result:
|
||||
|
||||
|
||||
7. visualize the result
|
||||
|
||||
|
||||
|
||||
|
||||
8. import VGG16 model
|
||||
|
||||
|
||||
|
||||
9. use conv_base model to extract features and labels
|
||||
|
||||
|
||||
10. training
|
||||
|
||||
|
||||
11. visualize the results
|
||||
|
||||
|
||||
12. build confusion matrix
|
||||
|
||||
|
||||
13. visualiz the confusion matrix
|
||||
|
||||
|
||||
14. sample Chinese multiperson voice spectrogram files are added
|
||||
15. sample VGG16 transfer learning code: vgg16.ipynb is added to the repository
|
||||
|
||||
|
||||
# Chinese-multiperson-voice-recognition-using-transfer-learning
|
||||
This is an example of applying transfer learning to the Chinese multi-person voice recognition application. Transfer learning is an AI technique used to enhance the training accuracy of use cases when the dataset is small or the training accuracy is low given the high noise of the original dataset. Multi-person voice recognition is known to contain high noise in the dataset. Chinese voice voice recognition has gained much progress recently thanks to the effort by the big name company such as Google. However many issues remain unsolved. Multi-person Chinese voice recognition is one of them. This example provieds not only multi-person Chinese voice sample dataset, but applied a transfer learning technique to the CNN trained model of the Chinese voice samples dataset. Satisfactory results can be achieved through transfer learning after an initial CNN training.
|
||||
This example provides a feasibility evidence of the transfer learning techniques, and it is our wish to convert the transfer learning technique to a Kubeflow asset through this illustration case. A transfer learning pipeline will be constructed to make kubeflow user easy to adapt to their model for training accuracy enhancement. Eventually, other users can benefit from such convenient features of the kubeflow resources.
|
||||
|
||||
usage briefing:
|
||||
1.Process audio files and convert them into spectrograms.
|
||||
2.Establish experimental data, divide them into 3 categories, and set them into CNN network training.
|
||||
3.Perform two training sessions to improve accuracy.
|
||||
4.Compare training methods.
|
||||
|
||||
Tools used:
|
||||
1. TensorFlow
|
||||
2. Anaconda
|
||||
3. Python3.7
|
||||
|
||||
1. preprocess(spectrograms production)
|
||||
|
||||
|
||||
|
||||
2. import spectrogram files.
|
||||
|
||||
|
||||
|
||||
|
||||
3. build training dataset:
|
||||
divide the dataset into training, validation, and testing sets.
|
||||
|
||||
|
||||
|
||||
4. build CNN taining:
|
||||
|
||||
|
||||
|
||||
5. first training
|
||||
|
||||
|
||||
|
||||
6. first training result:
|
||||
|
||||
|
||||
7. visualize the result
|
||||
|
||||
|
||||
|
||||
|
||||
8. import VGG16 model
|
||||
|
||||
|
||||
|
||||
9. use conv_base model to extract features and labels
|
||||
|
||||
|
||||
10. training
|
||||
|
||||
|
||||
11. visualize the results
|
||||
|
||||
|
||||
12. build confusion matrix
|
||||
|
||||
|
||||
13. visualiz the confusion matrix
|
||||
|
||||
|
||||
14. sample Chinese multiperson voice spectrogram files are added
|
||||
15. sample VGG16 transfer learning code: vgg16.ipynb is added to the repository
|
||||
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,43 +1,43 @@
|
|||
# Objective
|
||||
Here we convert the https://www.kaggle.com/competitions/facial-keypoints-detection code to kfp-pipeline
|
||||
The objective of this task is to predict keypoint positions on face images
|
||||
|
||||
# Testing enviornment
|
||||
The pipeline is tested on `Kubeflow 1.4` and `kfp 1.1.2` , it should be compatible with previous releases of Kubeflow . kfp version used for testing is 1.1.2 which can be installed as `pip install kfp==1.1.2`
|
||||
|
||||
# Components used
|
||||
|
||||
## Docker
|
||||
Docker is used to create an enviornment to run each component.
|
||||
|
||||
## Kubeflow pipelines
|
||||
Kubeflow pipelines connect each docker component and create a pipeline. Each Kubeflow pipeline is reproducable workflow wherein we pass input arguments and run entire workflow.
|
||||
|
||||
# Docker
|
||||
We start with creating a docker account on dockerhub (https://hub.docker.com/). We signup with our individual email. After signup is compelete login to docker using your username and password using the command `docker login` on your terminal
|
||||
|
||||
## Build train image
|
||||
Navigate to `train` directory, create a folder named `my_data` and put your `training.zip` and `test.zip` data from Kaggle repo in this folder and build docker image using :
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
In my case this is:
|
||||
```
|
||||
docker build -t hubdocker76/demotrain:v1 .
|
||||
```
|
||||
|
||||
## Build evaluate image
|
||||
Navigate to eval directory and build docker image using :
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
In my case this is:
|
||||
```
|
||||
docker build -t hubdocker76/demoeval:v2 .
|
||||
```
|
||||
# Kubeflow pipelines
|
||||
|
||||
Go to generate-pipeline and run `python3 my_pipeline.py` this will generate a yaml file. which we can upload to Kubeflow pipelines UI and create a Run from it.
|
||||
|
||||
# Sample pipeline to run on Kubeflow
|
||||
Navigate to directory `geneate-pipeline` and run `python3 my_pipeline.py` this will generate yaml file. I have named this yaml as `face_pipeline_01.yaml`. Please upload this pipeline on Kubeflow and start a Run.
|
||||
# Objective
|
||||
Here we convert the https://www.kaggle.com/competitions/facial-keypoints-detection code to kfp-pipeline
|
||||
The objective of this task is to predict keypoint positions on face images
|
||||
|
||||
# Testing enviornment
|
||||
The pipeline is tested on `Kubeflow 1.4` and `kfp 1.1.2` , it should be compatible with previous releases of Kubeflow . kfp version used for testing is 1.1.2 which can be installed as `pip install kfp==1.1.2`
|
||||
|
||||
# Components used
|
||||
|
||||
## Docker
|
||||
Docker is used to create an enviornment to run each component.
|
||||
|
||||
## Kubeflow pipelines
|
||||
Kubeflow pipelines connect each docker component and create a pipeline. Each Kubeflow pipeline is reproducable workflow wherein we pass input arguments and run entire workflow.
|
||||
|
||||
# Docker
|
||||
We start with creating a docker account on dockerhub (https://hub.docker.com/). We signup with our individual email. After signup is compelete login to docker using your username and password using the command `docker login` on your terminal
|
||||
|
||||
## Build train image
|
||||
Navigate to `train` directory, create a folder named `my_data` and put your `training.zip` and `test.zip` data from Kaggle repo in this folder and build docker image using :
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
In my case this is:
|
||||
```
|
||||
docker build -t hubdocker76/demotrain:v1 .
|
||||
```
|
||||
|
||||
## Build evaluate image
|
||||
Navigate to eval directory and build docker image using :
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
In my case this is:
|
||||
```
|
||||
docker build -t hubdocker76/demoeval:v2 .
|
||||
```
|
||||
# Kubeflow pipelines
|
||||
|
||||
Go to generate-pipeline and run `python3 my_pipeline.py` this will generate a yaml file. which we can upload to Kubeflow pipelines UI and create a Run from it.
|
||||
|
||||
# Sample pipeline to run on Kubeflow
|
||||
Navigate to directory `geneate-pipeline` and run `python3 my_pipeline.py` this will generate yaml file. I have named this yaml as `face_pipeline_01.yaml`. Please upload this pipeline on Kubeflow and start a Run.
|
||||
|
|
|
|||
|
|
@ -1,42 +1,42 @@
|
|||
import kfp
|
||||
from kfp import dsl
|
||||
|
||||
def SendMsg(trial, epoch, patience):
|
||||
vop = dsl.VolumeOp(name="pvc",
|
||||
resource_name="pvc", size='1Gi',
|
||||
modes=dsl.VOLUME_MODE_RWO)
|
||||
|
||||
return dsl.ContainerOp(
|
||||
name = 'Train',
|
||||
image = 'hubdocker76/demotrain:v1',
|
||||
command = ['python3', 'train.py'],
|
||||
arguments=[
|
||||
'--trial', trial,
|
||||
'--epoch', epoch,
|
||||
'--patience', patience
|
||||
],
|
||||
pvolumes={
|
||||
'/data': vop.volume
|
||||
}
|
||||
)
|
||||
|
||||
def GetMsg(comp1):
|
||||
return dsl.ContainerOp(
|
||||
name = 'Evaluate',
|
||||
image = 'hubdocker76/demoeval:v2',
|
||||
pvolumes={
|
||||
'/data': comp1.pvolumes['/data']
|
||||
},
|
||||
command = ['python3', 'eval.py']
|
||||
)
|
||||
|
||||
@dsl.pipeline(
|
||||
name = 'face pipeline',
|
||||
description = 'pipeline to detect facial landmarks')
|
||||
def passing_parameter(trial, epoch, patience):
|
||||
comp1 = SendMsg(trial, epoch, patience)
|
||||
comp2 = GetMsg(comp1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
import kfp.compiler as compiler
|
||||
compiler.Compiler().compile(passing_parameter, __file__ + '.yaml')
|
||||
import kfp
|
||||
from kfp import dsl
|
||||
|
||||
def SendMsg(trial, epoch, patience):
|
||||
vop = dsl.VolumeOp(name="pvc",
|
||||
resource_name="pvc", size='1Gi',
|
||||
modes=dsl.VOLUME_MODE_RWO)
|
||||
|
||||
return dsl.ContainerOp(
|
||||
name = 'Train',
|
||||
image = 'hubdocker76/demotrain:v1',
|
||||
command = ['python3', 'train.py'],
|
||||
arguments=[
|
||||
'--trial', trial,
|
||||
'--epoch', epoch,
|
||||
'--patience', patience
|
||||
],
|
||||
pvolumes={
|
||||
'/data': vop.volume
|
||||
}
|
||||
)
|
||||
|
||||
def GetMsg(comp1):
|
||||
return dsl.ContainerOp(
|
||||
name = 'Evaluate',
|
||||
image = 'hubdocker76/demoeval:v2',
|
||||
pvolumes={
|
||||
'/data': comp1.pvolumes['/data']
|
||||
},
|
||||
command = ['python3', 'eval.py']
|
||||
)
|
||||
|
||||
@dsl.pipeline(
|
||||
name = 'face pipeline',
|
||||
description = 'pipeline to detect facial landmarks')
|
||||
def passing_parameter(trial, epoch, patience):
|
||||
comp1 = SendMsg(trial, epoch, patience)
|
||||
comp2 = GetMsg(comp1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
import kfp.compiler as compiler
|
||||
compiler.Compiler().compile(passing_parameter, __file__ + '.yaml')
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,90 +1,90 @@
|
|||
body {
|
||||
background-color: azure;
|
||||
text-align: center;
|
||||
font-family: fantasy;
|
||||
}
|
||||
|
||||
#title {
|
||||
font-size: 10vh;
|
||||
}
|
||||
|
||||
#center {
|
||||
margin-left: 10%;
|
||||
margin-right: 10%;
|
||||
width: 80vw;
|
||||
height: 80vh;
|
||||
opacity: 2.0;
|
||||
background-color: blanchedalmond;
|
||||
border: 5px solid #666;
|
||||
border-radius: 50px;
|
||||
box-shadow:-9px 12px 9px black
|
||||
}
|
||||
|
||||
#input {
|
||||
float: left;
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
#output {
|
||||
float: right;
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.content-title {
|
||||
font-size: 5vh;
|
||||
margin: 2%;
|
||||
}
|
||||
|
||||
textarea {
|
||||
margin: 5%;
|
||||
width: 90%;
|
||||
font-size: 4vh;
|
||||
}
|
||||
|
||||
input {
|
||||
width: 30%;
|
||||
height: 10%;
|
||||
font-size: 4vh;
|
||||
font-family: fantasy;
|
||||
background-color: lawngreen;
|
||||
border-radius: 50px;
|
||||
}
|
||||
|
||||
.content-setting {
|
||||
font-size: 4vh;
|
||||
text-align: left;
|
||||
margin-left: 5%;
|
||||
}
|
||||
|
||||
#preprocess {
|
||||
border: 5px solid grey;
|
||||
border-radius: 10px;
|
||||
width: 87%;
|
||||
height: 25%;
|
||||
margin: 5%;
|
||||
background-color: white;
|
||||
font-size: 4vh;
|
||||
overflow: auto;
|
||||
word-break: break-all;
|
||||
word-wrap: break-word;
|
||||
text-align: left;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
table {
|
||||
border: 5px solid grey;
|
||||
border-radius: 10px;
|
||||
font-size: 4vh;
|
||||
margin: 5%;
|
||||
width: 90%;
|
||||
height: 25%;
|
||||
background-color: white;
|
||||
}
|
||||
|
||||
.predict-title {
|
||||
width: 33.33%;
|
||||
height: 20%;
|
||||
color: green;
|
||||
body {
|
||||
background-color: azure;
|
||||
text-align: center;
|
||||
font-family: fantasy;
|
||||
}
|
||||
|
||||
#title {
|
||||
font-size: 10vh;
|
||||
}
|
||||
|
||||
#center {
|
||||
margin-left: 10%;
|
||||
margin-right: 10%;
|
||||
width: 80vw;
|
||||
height: 80vh;
|
||||
opacity: 2.0;
|
||||
background-color: blanchedalmond;
|
||||
border: 5px solid #666;
|
||||
border-radius: 50px;
|
||||
box-shadow:-9px 12px 9px black
|
||||
}
|
||||
|
||||
#input {
|
||||
float: left;
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
#output {
|
||||
float: right;
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.content-title {
|
||||
font-size: 5vh;
|
||||
margin: 2%;
|
||||
}
|
||||
|
||||
textarea {
|
||||
margin: 5%;
|
||||
width: 90%;
|
||||
font-size: 4vh;
|
||||
}
|
||||
|
||||
input {
|
||||
width: 30%;
|
||||
height: 10%;
|
||||
font-size: 4vh;
|
||||
font-family: fantasy;
|
||||
background-color: lawngreen;
|
||||
border-radius: 50px;
|
||||
}
|
||||
|
||||
.content-setting {
|
||||
font-size: 4vh;
|
||||
text-align: left;
|
||||
margin-left: 5%;
|
||||
}
|
||||
|
||||
#preprocess {
|
||||
border: 5px solid grey;
|
||||
border-radius: 10px;
|
||||
width: 87%;
|
||||
height: 25%;
|
||||
margin: 5%;
|
||||
background-color: white;
|
||||
font-size: 4vh;
|
||||
overflow: auto;
|
||||
word-break: break-all;
|
||||
word-wrap: break-word;
|
||||
text-align: left;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
table {
|
||||
border: 5px solid grey;
|
||||
border-radius: 10px;
|
||||
font-size: 4vh;
|
||||
margin: 5%;
|
||||
width: 90%;
|
||||
height: 25%;
|
||||
background-color: white;
|
||||
}
|
||||
|
||||
.predict-title {
|
||||
width: 33.33%;
|
||||
height: 20%;
|
||||
color: green;
|
||||
}
|
||||
|
|
@ -1,156 +1,156 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Kubeflow - NLP</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="title">Kubeflow - NLP</div>
|
||||
<div id="center">
|
||||
<div id="input">
|
||||
<div class="content-title">Message</div>
|
||||
<form action="{{ url_for('predict')}}" method="POST" style="height: 70vh;">
|
||||
<textarea name="message" id="message" placeholder="...Please enter here" style="height: 60%;">{% print(message) %}</textarea>
|
||||
<input type="submit" value="Predict">
|
||||
</form>
|
||||
</div>
|
||||
<div id="output">
|
||||
<div class="content-title">Result</div>
|
||||
<div class="content-setting" style="color: red;">Positive +</div>
|
||||
<div class="content-setting" style="color: blue;">Negative -</div>
|
||||
<div id="preprocess">
|
||||
{% print(data) %}
|
||||
</div>
|
||||
<table border="1">
|
||||
<tr>
|
||||
<td class="predict-title">Numpy</td>
|
||||
<td class="predict-title">SKlearn</td>
|
||||
<td class="predict-title">Pytorch</td>
|
||||
<td class="predict-title">SVM</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
{% if my_prediction_np == 1%}
|
||||
<div style="font-size: 8vh; color: red;">+</div>
|
||||
{% elif my_prediction_np == 0%}
|
||||
<div style="font-size: 8vh; color: blue;">-</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if my_prediction_skl == 1%}
|
||||
<div style="font-size: 8vh; color: red;">+</div>
|
||||
{% elif my_prediction_skl == 0%}
|
||||
<div style="font-size: 8vh; color: blue;">-</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if my_prediction_toc == 1%}
|
||||
<div style="font-size: 8vh; color: red;">+</div>
|
||||
{% elif my_prediction_toc == 0%}
|
||||
<div style="font-size: 8vh; color: blue;">-</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if my_prediction_svm == 1%}
|
||||
<div style="font-size: 8vh; color: red;">+</div>
|
||||
{% elif my_prediction_svm == 0%}
|
||||
<div style="font-size: 8vh; color: blue;">-</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
<style type="text/css">
|
||||
body {
|
||||
background-color: azure;
|
||||
text-align: center;
|
||||
font-family: fantasy;
|
||||
}
|
||||
|
||||
#title {
|
||||
font-size: 10vh;
|
||||
}
|
||||
|
||||
#center {
|
||||
margin-left: 10%;
|
||||
margin-right: 10%;
|
||||
width: 80vw;
|
||||
height: 80vh;
|
||||
opacity: 2.0;
|
||||
background-color: blanchedalmond;
|
||||
border: 5px solid #666;
|
||||
border-radius: 50px;
|
||||
box-shadow:-9px 12px 9px black
|
||||
}
|
||||
|
||||
#input {
|
||||
float: left;
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
#output {
|
||||
float: right;
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.content-title {
|
||||
font-size: 5vh;
|
||||
margin: 2%;
|
||||
}
|
||||
|
||||
textarea {
|
||||
margin: 5%;
|
||||
width: 90%;
|
||||
font-size: 4vh;
|
||||
}
|
||||
|
||||
input {
|
||||
width: 30%;
|
||||
height: 10%;
|
||||
font-size: 4vh;
|
||||
font-family: fantasy;
|
||||
background-color: lawngreen;
|
||||
border-radius: 50px;
|
||||
}
|
||||
|
||||
.content-setting {
|
||||
font-size: 4vh;
|
||||
text-align: left;
|
||||
margin-left: 5%;
|
||||
}
|
||||
|
||||
#preprocess {
|
||||
border: 5px solid grey;
|
||||
border-radius: 10px;
|
||||
width: 88.1%;
|
||||
height: 25%;
|
||||
margin: 5%;
|
||||
background-color: white;
|
||||
font-size: 4vh;
|
||||
overflow: auto;
|
||||
word-break: break-all;
|
||||
word-wrap: break-word;
|
||||
text-align: left;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
table {
|
||||
border: 5px solid grey;
|
||||
border-radius: 10px;
|
||||
font-size: 4vh;
|
||||
margin: 5%;
|
||||
width: 90%;
|
||||
height: 25%;
|
||||
background-color: white;
|
||||
}
|
||||
|
||||
.predict-title {
|
||||
width: 25%;
|
||||
height: 20%;
|
||||
color: green;
|
||||
}
|
||||
</style>
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Kubeflow - NLP</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="title">Kubeflow - NLP</div>
|
||||
<div id="center">
|
||||
<div id="input">
|
||||
<div class="content-title">Message</div>
|
||||
<form action="{{ url_for('predict')}}" method="POST" style="height: 70vh;">
|
||||
<textarea name="message" id="message" placeholder="...Please enter here" style="height: 60%;">{% print(message) %}</textarea>
|
||||
<input type="submit" value="Predict">
|
||||
</form>
|
||||
</div>
|
||||
<div id="output">
|
||||
<div class="content-title">Result</div>
|
||||
<div class="content-setting" style="color: red;">Positive +</div>
|
||||
<div class="content-setting" style="color: blue;">Negative -</div>
|
||||
<div id="preprocess">
|
||||
{% print(data) %}
|
||||
</div>
|
||||
<table border="1">
|
||||
<tr>
|
||||
<td class="predict-title">Numpy</td>
|
||||
<td class="predict-title">SKlearn</td>
|
||||
<td class="predict-title">Pytorch</td>
|
||||
<td class="predict-title">SVM</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
{% if my_prediction_np == 1%}
|
||||
<div style="font-size: 8vh; color: red;">+</div>
|
||||
{% elif my_prediction_np == 0%}
|
||||
<div style="font-size: 8vh; color: blue;">-</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if my_prediction_skl == 1%}
|
||||
<div style="font-size: 8vh; color: red;">+</div>
|
||||
{% elif my_prediction_skl == 0%}
|
||||
<div style="font-size: 8vh; color: blue;">-</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if my_prediction_toc == 1%}
|
||||
<div style="font-size: 8vh; color: red;">+</div>
|
||||
{% elif my_prediction_toc == 0%}
|
||||
<div style="font-size: 8vh; color: blue;">-</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if my_prediction_svm == 1%}
|
||||
<div style="font-size: 8vh; color: red;">+</div>
|
||||
{% elif my_prediction_svm == 0%}
|
||||
<div style="font-size: 8vh; color: blue;">-</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
<style type="text/css">
|
||||
body {
|
||||
background-color: azure;
|
||||
text-align: center;
|
||||
font-family: fantasy;
|
||||
}
|
||||
|
||||
#title {
|
||||
font-size: 10vh;
|
||||
}
|
||||
|
||||
#center {
|
||||
margin-left: 10%;
|
||||
margin-right: 10%;
|
||||
width: 80vw;
|
||||
height: 80vh;
|
||||
opacity: 2.0;
|
||||
background-color: blanchedalmond;
|
||||
border: 5px solid #666;
|
||||
border-radius: 50px;
|
||||
box-shadow:-9px 12px 9px black
|
||||
}
|
||||
|
||||
#input {
|
||||
float: left;
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
#output {
|
||||
float: right;
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.content-title {
|
||||
font-size: 5vh;
|
||||
margin: 2%;
|
||||
}
|
||||
|
||||
textarea {
|
||||
margin: 5%;
|
||||
width: 90%;
|
||||
font-size: 4vh;
|
||||
}
|
||||
|
||||
input {
|
||||
width: 30%;
|
||||
height: 10%;
|
||||
font-size: 4vh;
|
||||
font-family: fantasy;
|
||||
background-color: lawngreen;
|
||||
border-radius: 50px;
|
||||
}
|
||||
|
||||
.content-setting {
|
||||
font-size: 4vh;
|
||||
text-align: left;
|
||||
margin-left: 5%;
|
||||
}
|
||||
|
||||
#preprocess {
|
||||
border: 5px solid grey;
|
||||
border-radius: 10px;
|
||||
width: 88.1%;
|
||||
height: 25%;
|
||||
margin: 5%;
|
||||
background-color: white;
|
||||
font-size: 4vh;
|
||||
overflow: auto;
|
||||
word-break: break-all;
|
||||
word-wrap: break-word;
|
||||
text-align: left;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
table {
|
||||
border: 5px solid grey;
|
||||
border-radius: 10px;
|
||||
font-size: 4vh;
|
||||
margin: 5%;
|
||||
width: 90%;
|
||||
height: 25%;
|
||||
background-color: white;
|
||||
}
|
||||
|
||||
.predict-title {
|
||||
width: 25%;
|
||||
height: 20%;
|
||||
color: green;
|
||||
}
|
||||
</style>
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.5 KiB |
|
|
@ -0,0 +1,24 @@
|
|||
FROM nodered/node-red:2.2.3-12
|
||||
|
||||
ARG PREFIX
|
||||
|
||||
ENV NODE_OPTIONS=--max_old_space_size=128
|
||||
|
||||
USER root
|
||||
RUN apk update && \
|
||||
apk add py3-pip python3-dev libffi-dev
|
||||
|
||||
USER root
|
||||
RUN apk update && \
|
||||
apk add py3-pip
|
||||
RUN apk update && apk add python3-dev
|
||||
USER node-red
|
||||
|
||||
# Copy package.json to the WORKDIR so npm builds all
|
||||
# of your added nodes modules for Node-RED
|
||||
COPY package.json .
|
||||
RUN npm install --unsafe-perm --no-update-notifier --no-fund --only=production
|
||||
|
||||
ADD scripts/entrypoint.sh .
|
||||
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
docker compose build
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1 @@
|
|||
{"version":"1","format":"xl-single","id":"fd84d37c-ca41-455e-91a5-092551ea2351","xl":{"version":"3","this":"b6e74063-d6f7-410f-ad3f-4e8fe53877e1","sets":[["b6e74063-d6f7-410f-ad3f-4e8fe53877e1"]],"distributionAlgo":"SIPMOD+PARITY"}}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,43 @@
|
|||
version: '3.7'
|
||||
|
||||
services:
|
||||
node-red:
|
||||
image: reg.footprint-ai.com/public/kube-nodered:latest
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
labels:
|
||||
com.docker.compose.container-number: "1"
|
||||
environment:
|
||||
- TZ=Asia/Taipei
|
||||
- USERDIR=$USERDIR
|
||||
- NODE_RED_CREDENTIAL_SECRET=noderedtutorial
|
||||
- KUBEFLOW_HOST=$KUBEFLOW_HOST
|
||||
- KUBEFLOW_USERNAME=$KUBEFLOW_USERNAME
|
||||
- KUBEFLOW_PASSWORD=$KUBEFLOW_PASSWORD
|
||||
- MINIO_HOST=$MINIO_HOST
|
||||
- MINIO_ROOT_USER=$MINIO_ROOT_USER
|
||||
- MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
|
||||
ports:
|
||||
- "1880:1880"
|
||||
networks:
|
||||
- node-red-net
|
||||
volumes:
|
||||
- ./:/data
|
||||
|
||||
minio:
|
||||
image: minio/minio
|
||||
command: server --console-address ":9001" /data
|
||||
expose:
|
||||
- "9000"
|
||||
environment:
|
||||
- MINIO_ROOT_USER=$MINIO_ROOT_USER
|
||||
- MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
|
||||
ports:
|
||||
- "9001:9001"
|
||||
- "9000:9000"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
|
||||
networks:
|
||||
node-red-net:
|
||||
|
|
@ -0,0 +1,446 @@
|
|||
{
|
||||
"node-red": {
|
||||
"name": "node-red",
|
||||
"version": "2.2.3",
|
||||
"local": false,
|
||||
"user": false,
|
||||
"nodes": {
|
||||
"inject": {
|
||||
"name": "inject",
|
||||
"types": [
|
||||
"inject"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/20-inject.js"
|
||||
},
|
||||
"debug": {
|
||||
"name": "debug",
|
||||
"types": [
|
||||
"debug"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/21-debug.js"
|
||||
},
|
||||
"complete": {
|
||||
"name": "complete",
|
||||
"types": [
|
||||
"complete"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/24-complete.js"
|
||||
},
|
||||
"catch": {
|
||||
"name": "catch",
|
||||
"types": [
|
||||
"catch"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/25-catch.js"
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"types": [
|
||||
"status"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/25-status.js"
|
||||
},
|
||||
"link": {
|
||||
"name": "link",
|
||||
"types": [
|
||||
"link in",
|
||||
"link out",
|
||||
"link call"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/60-link.js"
|
||||
},
|
||||
"comment": {
|
||||
"name": "comment",
|
||||
"types": [
|
||||
"comment"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/90-comment.js"
|
||||
},
|
||||
"unknown": {
|
||||
"name": "unknown",
|
||||
"types": [
|
||||
"unknown"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/98-unknown.js"
|
||||
},
|
||||
"function": {
|
||||
"name": "function",
|
||||
"types": [
|
||||
"function"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/10-function.js"
|
||||
},
|
||||
"switch": {
|
||||
"name": "switch",
|
||||
"types": [
|
||||
"switch"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/10-switch.js"
|
||||
},
|
||||
"change": {
|
||||
"name": "change",
|
||||
"types": [
|
||||
"change"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/15-change.js"
|
||||
},
|
||||
"range": {
|
||||
"name": "range",
|
||||
"types": [
|
||||
"range"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/16-range.js"
|
||||
},
|
||||
"template": {
|
||||
"name": "template",
|
||||
"types": [
|
||||
"template"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/80-template.js"
|
||||
},
|
||||
"delay": {
|
||||
"name": "delay",
|
||||
"types": [
|
||||
"delay"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/89-delay.js"
|
||||
},
|
||||
"trigger": {
|
||||
"name": "trigger",
|
||||
"types": [
|
||||
"trigger"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/89-trigger.js"
|
||||
},
|
||||
"exec": {
|
||||
"name": "exec",
|
||||
"types": [
|
||||
"exec"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/90-exec.js"
|
||||
},
|
||||
"rbe": {
|
||||
"name": "rbe",
|
||||
"types": [
|
||||
"rbe"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/rbe.js"
|
||||
},
|
||||
"tls": {
|
||||
"name": "tls",
|
||||
"types": [
|
||||
"tls-config"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/05-tls.js"
|
||||
},
|
||||
"httpproxy": {
|
||||
"name": "httpproxy",
|
||||
"types": [
|
||||
"http proxy"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/06-httpproxy.js"
|
||||
},
|
||||
"mqtt": {
|
||||
"name": "mqtt",
|
||||
"types": [
|
||||
"mqtt in",
|
||||
"mqtt out",
|
||||
"mqtt-broker"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/10-mqtt.js"
|
||||
},
|
||||
"httpin": {
|
||||
"name": "httpin",
|
||||
"types": [
|
||||
"http in",
|
||||
"http response"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/21-httpin.js"
|
||||
},
|
||||
"httprequest": {
|
||||
"name": "httprequest",
|
||||
"types": [
|
||||
"http request"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/21-httprequest.js"
|
||||
},
|
||||
"websocket": {
|
||||
"name": "websocket",
|
||||
"types": [
|
||||
"websocket in",
|
||||
"websocket out",
|
||||
"websocket-listener",
|
||||
"websocket-client"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/22-websocket.js"
|
||||
},
|
||||
"tcpin": {
|
||||
"name": "tcpin",
|
||||
"types": [
|
||||
"tcp in",
|
||||
"tcp out",
|
||||
"tcp request"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/31-tcpin.js"
|
||||
},
|
||||
"udp": {
|
||||
"name": "udp",
|
||||
"types": [
|
||||
"udp in",
|
||||
"udp out"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/32-udp.js"
|
||||
},
|
||||
"CSV": {
|
||||
"name": "CSV",
|
||||
"types": [
|
||||
"csv"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-CSV.js"
|
||||
},
|
||||
"HTML": {
|
||||
"name": "HTML",
|
||||
"types": [
|
||||
"html"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-HTML.js"
|
||||
},
|
||||
"JSON": {
|
||||
"name": "JSON",
|
||||
"types": [
|
||||
"json"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-JSON.js"
|
||||
},
|
||||
"XML": {
|
||||
"name": "XML",
|
||||
"types": [
|
||||
"xml"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-XML.js"
|
||||
},
|
||||
"YAML": {
|
||||
"name": "YAML",
|
||||
"types": [
|
||||
"yaml"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-YAML.js"
|
||||
},
|
||||
"split": {
|
||||
"name": "split",
|
||||
"types": [
|
||||
"split",
|
||||
"join"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/17-split.js"
|
||||
},
|
||||
"sort": {
|
||||
"name": "sort",
|
||||
"types": [
|
||||
"sort"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/18-sort.js"
|
||||
},
|
||||
"batch": {
|
||||
"name": "batch",
|
||||
"types": [
|
||||
"batch"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/19-batch.js"
|
||||
},
|
||||
"file": {
|
||||
"name": "file",
|
||||
"types": [
|
||||
"file",
|
||||
"file in"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/storage/10-file.js"
|
||||
},
|
||||
"watch": {
|
||||
"name": "watch",
|
||||
"types": [
|
||||
"watch"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red",
|
||||
"file": "/usr/src/node-red/node_modules/@node-red/nodes/core/storage/23-watch.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"node-red-contrib-autoai-pipeline": {
|
||||
"name": "node-red-contrib-autoai-pipeline",
|
||||
"version": "0.0.1",
|
||||
"local": true,
|
||||
"user": false,
|
||||
"nodes": {
|
||||
"fl": {
|
||||
"name": "fl",
|
||||
"types": [
|
||||
"RAG"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": true,
|
||||
"user": false,
|
||||
"module": "node-red-contrib-autoai-pipeline",
|
||||
"file": "/data/main/node_modules/autoAI pipeline/RAG/RAG.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"node-red-contrib-pythonshell": {
|
||||
"name": "node-red-contrib-pythonshell",
|
||||
"version": "1.5.4",
|
||||
"local": false,
|
||||
"user": false,
|
||||
"nodes": {
|
||||
"pythonshell": {
|
||||
"name": "pythonshell",
|
||||
"types": [
|
||||
"pythonshell in"
|
||||
],
|
||||
"enabled": true,
|
||||
"local": false,
|
||||
"user": false,
|
||||
"module": "node-red-contrib-pythonshell",
|
||||
"file": "/usr/src/node-red/node_modules/node-red-contrib-pythonshell/src/pythonshell.js"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"_": {
|
||||
"editor": {
|
||||
"view": {
|
||||
"view-store-zoom": false,
|
||||
"view-store-position": false,
|
||||
"view-show-grid": true,
|
||||
"view-snap-grid": true,
|
||||
"view-grid-size": 20,
|
||||
"view-node-status": true,
|
||||
"view-node-show-label": true,
|
||||
"view-show-tips": true,
|
||||
"view-show-welcome-tours": true
|
||||
},
|
||||
"tours": {
|
||||
"welcome": "2.2.3"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"_": {
|
||||
"editor": {
|
||||
"view": {
|
||||
"view-store-zoom": false,
|
||||
"view-store-position": false,
|
||||
"view-show-grid": true,
|
||||
"view-snap-grid": true,
|
||||
"view-grid-size": 20,
|
||||
"view-node-status": true,
|
||||
"view-node-show-label": true,
|
||||
"view-show-tips": true,
|
||||
"view-show-welcome-tours": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,650 @@
|
|||
[
|
||||
{
|
||||
"id": "f6f2187d.f17ca8",
|
||||
"type": "tab",
|
||||
"label": "Flow 1",
|
||||
"disabled": false,
|
||||
"info": ""
|
||||
},
|
||||
{
|
||||
"id": "c9fd2f76f856879d",
|
||||
"type": "tab",
|
||||
"label": "base flow",
|
||||
"disabled": false,
|
||||
"info": "",
|
||||
"env": []
|
||||
},
|
||||
{
|
||||
"id": "e0941514f4cd8be3",
|
||||
"type": "tab",
|
||||
"label": "流程1",
|
||||
"disabled": false,
|
||||
"info": "",
|
||||
"env": []
|
||||
},
|
||||
{
|
||||
"id": "3cc11d24.ff01a2",
|
||||
"type": "comment",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "WARNING: please check you have started this container with a volume that is mounted to /data\\n otherwise any flow changes are lost when you redeploy or upgrade the container\\n (e.g. upgrade to a more recent node-red docker image).\\n If you are using named volumes you can ignore this warning.\\n Double click or see info side panel to learn how to start Node-RED in Docker to save your work",
|
||||
"info": "\nTo start docker with a bind mount volume (-v option), for example:\n\n```\ndocker run -it -p 1880:1880 -v /home/user/node_red_data:/data --name mynodered nodered/node-red\n```\n\nwhere `/home/user/node_red_data` is a directory on your host machine where you want to store your flows.\n\nIf you do not do this then you can experiment and redploy flows, but if you restart or upgrade the container the flows will be disconnected and lost. \n\nThey will still exist in a hidden data volume, which can be recovered using standard docker techniques, but that is much more complex than just starting with a named volume as described above.",
|
||||
"x": 350,
|
||||
"y": 80,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "b126ea03f7d74573",
|
||||
"type": "debug",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 450,
|
||||
"y": 440,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "692c86b087eed4a0",
|
||||
"type": "pythonshell in",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "install dependency",
|
||||
"pyfile": "/data/main/py/install.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 270,
|
||||
"y": 300,
|
||||
"wires": [
|
||||
[
|
||||
"0b5c3b39a424dc6a"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "0b5c3b39a424dc6a",
|
||||
"type": "debug",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": true,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "payload",
|
||||
"statusType": "auto",
|
||||
"x": 470,
|
||||
"y": 300,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "3b1a0675389769a1",
|
||||
"type": "comment",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "install python dependency",
|
||||
"info": "",
|
||||
"x": 370,
|
||||
"y": 240,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "04e11dfd70107dcf",
|
||||
"type": "comment",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "read experiemnt data from kfp",
|
||||
"info": "",
|
||||
"x": 380,
|
||||
"y": 380,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "ffe395298ffe40e0",
|
||||
"type": "pythonshell in",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "list experiments",
|
||||
"pyfile": "/data/main/py/api_examples/list_experiments.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 260,
|
||||
"y": 440,
|
||||
"wires": [
|
||||
[
|
||||
"b126ea03f7d74573"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "d7c0bb24de178d22",
|
||||
"type": "RAG",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "",
|
||||
"url": "",
|
||||
"data_name": "",
|
||||
"huggingface_token": "",
|
||||
"embedding_model": "",
|
||||
"LLM_model": "",
|
||||
"Pinecone_token": "",
|
||||
"Pinecone_index": "",
|
||||
"host": "",
|
||||
"username": "",
|
||||
"userpassword": "",
|
||||
"x": 390,
|
||||
"y": 580,
|
||||
"wires": [
|
||||
[]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "b33d174d94620977",
|
||||
"type": "debug",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "false",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 730,
|
||||
"y": 240,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "6b5130cec3efaf0a",
|
||||
"type": "debug",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "false",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 320,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "84f40c6e98828bb5",
|
||||
"type": "debug",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 770,
|
||||
"y": 400,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "1a03c8eab24f2d9d",
|
||||
"type": "pythonshell in",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "test",
|
||||
"pyfile": "/data/main/py/model35.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 510,
|
||||
"y": 520,
|
||||
"wires": [
|
||||
[
|
||||
"241bb407c3b1a512"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "241bb407c3b1a512",
|
||||
"type": "debug",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 730,
|
||||
"y": 520,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "9211da9222209e48",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 140,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "f2394d82583d42d2",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "install dependency",
|
||||
"pyfile": "/data/main/py/install.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 550,
|
||||
"y": 80,
|
||||
"wires": [
|
||||
[
|
||||
"cec42d56788a6d86"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "cec42d56788a6d86",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": true,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "payload",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 80,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "6df33a3d52decf56",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "list experiments",
|
||||
"pyfile": "/data/main/py/api_examples/list_experiments.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 140,
|
||||
"wires": [
|
||||
[
|
||||
"9211da9222209e48"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "20410c4e397c7805",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "list_pipelines",
|
||||
"pyfile": "/data/main/py/api_examples/list_pipelines.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 530,
|
||||
"y": 200,
|
||||
"wires": [
|
||||
[
|
||||
"8400360595296c8f"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "8400360595296c8f",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 200,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "20ae5c175bd4bd4d",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "upload_pipeline",
|
||||
"pyfile": "/data/main/py/api_examples/upload_pipeline.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 260,
|
||||
"wires": [
|
||||
[
|
||||
"711b376a156b58a2"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "711b376a156b58a2",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 260,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "5dc55b480c912ba0",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "create_experiment",
|
||||
"pyfile": "/data/main/py/api_examples/create_experiment.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 550,
|
||||
"y": 320,
|
||||
"wires": [
|
||||
[
|
||||
"47ae84a2b85aec74"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "47ae84a2b85aec74",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 320,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "bf79cd5247e6d498",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "create_run.py",
|
||||
"pyfile": "/data/main/py/api_examples/create_run.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 380,
|
||||
"wires": [
|
||||
[
|
||||
"41280a3e2bd16bf1"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "41280a3e2bd16bf1",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 380,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "9c2a05f63fa71bc7",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "delete_experiment",
|
||||
"pyfile": "/data/main/py/api_examples/delete_experiment.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 550,
|
||||
"y": 440,
|
||||
"wires": [
|
||||
[
|
||||
"5a77999e669b4c06"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "5a77999e669b4c06",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 440,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "ef4b5e874d4c37ab",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "delete_pipeline",
|
||||
"pyfile": "/data/main/py/api_examples/delete_pipeline.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 500,
|
||||
"wires": [
|
||||
[
|
||||
"29e24d1ea757f2c6"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "29e24d1ea757f2c6",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 500,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "a2c4616e7a1591b7",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "delete_run.py",
|
||||
"pyfile": "/data/main/py/api_examples/delete_run.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 560,
|
||||
"wires": [
|
||||
[
|
||||
"5f127d565442e111"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "5f127d565442e111",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 560,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "80de59953d4b8556",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "get_run",
|
||||
"pyfile": "/data/main/py/api_examples/get_run.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 520,
|
||||
"y": 620,
|
||||
"wires": [
|
||||
[
|
||||
"4ccad67787ec0207"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "4ccad67787ec0207",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 620,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "45cde0d735c7fa70",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "kfp_login.py",
|
||||
"pyfile": "/data/main/py/api_examples/kfp_login.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 530,
|
||||
"y": 680,
|
||||
"wires": [
|
||||
[
|
||||
"9f35716a8bed686e"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "9f35716a8bed686e",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 680,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "b33bc4ad7e423460",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "kfp_login.py",
|
||||
"pyfile": "/data/main/py/api_examples/kfp_namespace.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 530,
|
||||
"y": 740,
|
||||
"wires": [
|
||||
[
|
||||
"a139fa9944b79f94"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "a139fa9944b79f94",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 740,
|
||||
"wires": []
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,697 @@
|
|||
[
|
||||
{
|
||||
"id": "f6f2187d.f17ca8",
|
||||
"type": "tab",
|
||||
"label": "Flow 1",
|
||||
"disabled": false,
|
||||
"info": ""
|
||||
},
|
||||
{
|
||||
"id": "c9fd2f76f856879d",
|
||||
"type": "tab",
|
||||
"label": "base flow",
|
||||
"disabled": false,
|
||||
"info": "",
|
||||
"env": []
|
||||
},
|
||||
{
|
||||
"id": "e0941514f4cd8be3",
|
||||
"type": "tab",
|
||||
"label": "流程1",
|
||||
"disabled": false,
|
||||
"info": "",
|
||||
"env": []
|
||||
},
|
||||
{
|
||||
"id": "3cc11d24.ff01a2",
|
||||
"type": "comment",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "WARNING: please check you have started this container with a volume that is mounted to /data\\n otherwise any flow changes are lost when you redeploy or upgrade the container\\n (e.g. upgrade to a more recent node-red docker image).\\n If you are using named volumes you can ignore this warning.\\n Double click or see info side panel to learn how to start Node-RED in Docker to save your work",
|
||||
"info": "\nTo start docker with a bind mount volume (-v option), for example:\n\n```\ndocker run -it -p 1880:1880 -v /home/user/node_red_data:/data --name mynodered nodered/node-red\n```\n\nwhere `/home/user/node_red_data` is a directory on your host machine where you want to store your flows.\n\nIf you do not do this then you can experiment and redploy flows, but if you restart or upgrade the container the flows will be disconnected and lost. \n\nThey will still exist in a hidden data volume, which can be recovered using standard docker techniques, but that is much more complex than just starting with a named volume as described above.",
|
||||
"x": 350,
|
||||
"y": 80,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "b126ea03f7d74573",
|
||||
"type": "debug",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 450,
|
||||
"y": 440,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "692c86b087eed4a0",
|
||||
"type": "pythonshell in",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "install dependency",
|
||||
"pyfile": "/data/main/py/install.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 270,
|
||||
"y": 300,
|
||||
"wires": [
|
||||
[
|
||||
"0b5c3b39a424dc6a"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "0b5c3b39a424dc6a",
|
||||
"type": "debug",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": true,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "payload",
|
||||
"statusType": "auto",
|
||||
"x": 470,
|
||||
"y": 300,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "3b1a0675389769a1",
|
||||
"type": "comment",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "install python dependency",
|
||||
"info": "",
|
||||
"x": 370,
|
||||
"y": 240,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "04e11dfd70107dcf",
|
||||
"type": "comment",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "read experiemnt data from kfp",
|
||||
"info": "",
|
||||
"x": 380,
|
||||
"y": 380,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "ffe395298ffe40e0",
|
||||
"type": "pythonshell in",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "list experiments",
|
||||
"pyfile": "/data/main/py/api_examples/list_experiments.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 260,
|
||||
"y": 440,
|
||||
"wires": [
|
||||
[
|
||||
"b126ea03f7d74573"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "d7c0bb24de178d22",
|
||||
"type": "RAG",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "",
|
||||
"url": "https://arxiv.org/pdf/2207.02696.pdf",
|
||||
"data_name": "/yolov7.pdf",
|
||||
"huggingface_token": "",
|
||||
"embedding_model": "sentence-transformers/all-MiniLM-L6-v2",
|
||||
"LLM_model": "openai-community/gpt2",
|
||||
"Pinecone_token": "",
|
||||
"Pinecone_index": "test1",
|
||||
"host": "",
|
||||
"username": "",
|
||||
"userpassword": "",
|
||||
"x": 390,
|
||||
"y": 580,
|
||||
"wires": [
|
||||
[
|
||||
"a4743858ec1f39eb"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "e659035ebcd2420d",
|
||||
"type": "inject",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "",
|
||||
"props": [
|
||||
{
|
||||
"p": "payload"
|
||||
},
|
||||
{
|
||||
"p": "topic",
|
||||
"vt": "str"
|
||||
}
|
||||
],
|
||||
"repeat": "",
|
||||
"crontab": "",
|
||||
"once": false,
|
||||
"onceDelay": 0.1,
|
||||
"topic": "",
|
||||
"payload": "",
|
||||
"payloadType": "date",
|
||||
"x": 250,
|
||||
"y": 580,
|
||||
"wires": [
|
||||
[
|
||||
"d7c0bb24de178d22"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "a4743858ec1f39eb",
|
||||
"type": "debug",
|
||||
"z": "f6f2187d.f17ca8",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "false",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 530,
|
||||
"y": 580,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "b33d174d94620977",
|
||||
"type": "debug",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "false",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 730,
|
||||
"y": 240,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "6b5130cec3efaf0a",
|
||||
"type": "debug",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "false",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 320,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "84f40c6e98828bb5",
|
||||
"type": "debug",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 770,
|
||||
"y": 400,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "1a03c8eab24f2d9d",
|
||||
"type": "pythonshell in",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "test",
|
||||
"pyfile": "/data/main/py/model35.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 510,
|
||||
"y": 520,
|
||||
"wires": [
|
||||
[
|
||||
"241bb407c3b1a512"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "241bb407c3b1a512",
|
||||
"type": "debug",
|
||||
"z": "c9fd2f76f856879d",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 730,
|
||||
"y": 520,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "9211da9222209e48",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 140,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "f2394d82583d42d2",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "install dependency",
|
||||
"pyfile": "/data/main/py/install.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 550,
|
||||
"y": 80,
|
||||
"wires": [
|
||||
[
|
||||
"cec42d56788a6d86"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "cec42d56788a6d86",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": true,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "payload",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 80,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "6df33a3d52decf56",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "list experiments",
|
||||
"pyfile": "/data/main/py/api_examples/list_experiments.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 140,
|
||||
"wires": [
|
||||
[
|
||||
"9211da9222209e48"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "20410c4e397c7805",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "list_pipelines",
|
||||
"pyfile": "/data/main/py/api_examples/list_pipelines.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 530,
|
||||
"y": 200,
|
||||
"wires": [
|
||||
[
|
||||
"8400360595296c8f"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "8400360595296c8f",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 200,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "20ae5c175bd4bd4d",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "upload_pipeline",
|
||||
"pyfile": "/data/main/py/api_examples/upload_pipeline.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 260,
|
||||
"wires": [
|
||||
[
|
||||
"711b376a156b58a2"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "711b376a156b58a2",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 260,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "5dc55b480c912ba0",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "create_experiment",
|
||||
"pyfile": "/data/main/py/api_examples/create_experiment.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 550,
|
||||
"y": 320,
|
||||
"wires": [
|
||||
[
|
||||
"47ae84a2b85aec74"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "47ae84a2b85aec74",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 320,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "bf79cd5247e6d498",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "create_run.py",
|
||||
"pyfile": "/data/main/py/api_examples/create_run.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 380,
|
||||
"wires": [
|
||||
[
|
||||
"41280a3e2bd16bf1"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "41280a3e2bd16bf1",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 380,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "9c2a05f63fa71bc7",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "delete_experiment",
|
||||
"pyfile": "/data/main/py/api_examples/delete_experiment.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 550,
|
||||
"y": 440,
|
||||
"wires": [
|
||||
[
|
||||
"5a77999e669b4c06"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "5a77999e669b4c06",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 440,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "ef4b5e874d4c37ab",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "delete_pipeline",
|
||||
"pyfile": "/data/main/py/api_examples/delete_pipeline.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 500,
|
||||
"wires": [
|
||||
[
|
||||
"29e24d1ea757f2c6"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "29e24d1ea757f2c6",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 500,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "a2c4616e7a1591b7",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "delete_run.py",
|
||||
"pyfile": "/data/main/py/api_examples/delete_run.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 540,
|
||||
"y": 560,
|
||||
"wires": [
|
||||
[
|
||||
"5f127d565442e111"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "5f127d565442e111",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 560,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "80de59953d4b8556",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "get_run",
|
||||
"pyfile": "/data/main/py/api_examples/get_run.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 520,
|
||||
"y": 620,
|
||||
"wires": [
|
||||
[
|
||||
"4ccad67787ec0207"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "4ccad67787ec0207",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 620,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "45cde0d735c7fa70",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "kfp_login.py",
|
||||
"pyfile": "/data/main/py/api_examples/kfp_login.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 530,
|
||||
"y": 680,
|
||||
"wires": [
|
||||
[
|
||||
"9f35716a8bed686e"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "9f35716a8bed686e",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 680,
|
||||
"wires": []
|
||||
},
|
||||
{
|
||||
"id": "b33bc4ad7e423460",
|
||||
"type": "pythonshell in",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "kfp_login.py",
|
||||
"pyfile": "/data/main/py/api_examples/kfp_namespace.py",
|
||||
"virtualenv": "",
|
||||
"continuous": true,
|
||||
"stdInData": false,
|
||||
"python3": true,
|
||||
"x": 530,
|
||||
"y": 740,
|
||||
"wires": [
|
||||
[
|
||||
"a139fa9944b79f94"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "a139fa9944b79f94",
|
||||
"type": "debug",
|
||||
"z": "e0941514f4cd8be3",
|
||||
"name": "",
|
||||
"active": true,
|
||||
"tosidebar": true,
|
||||
"console": false,
|
||||
"tostatus": false,
|
||||
"complete": "payload",
|
||||
"targetType": "msg",
|
||||
"statusVal": "",
|
||||
"statusType": "auto",
|
||||
"x": 750,
|
||||
"y": 740,
|
||||
"wires": []
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"$": "debb1e3e4666ba98dd5189a6e20b7e40jk4="
|
||||
}
|
||||
91
RAG-pipeline-with-nodered/RAG_with_node-red/example/main/node_modules/autoAI pipeline/RAG/RAG.html
generated
vendored
Normal file
91
RAG-pipeline-with-nodered/RAG_with_node-red/example/main/node_modules/autoAI pipeline/RAG/RAG.html
generated
vendored
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
<script type="text/javascript">
|
||||
RED.nodes.registerType('RAG', {
|
||||
category: 'function',
|
||||
color: '#7F8C8D',
|
||||
defaults: {
|
||||
name: { value: "" },
|
||||
url:{value:""},
|
||||
data_name:{value:""},
|
||||
huggingface_token:{value:""},
|
||||
embedding_model:{value:""},
|
||||
LLM_model:{value:""},
|
||||
Pinecone_token:{value:""},
|
||||
Pinecone_index:{value:""},
|
||||
|
||||
host:{value:""},
|
||||
username:{value:""},
|
||||
userpassword:{value:""},
|
||||
|
||||
},
|
||||
inputs: 1,
|
||||
outputs: 1,
|
||||
icon: "font-awesome/fa-file-text",
|
||||
label: function() {
|
||||
return this.name ||"RAG";
|
||||
},
|
||||
oneditprepare:{
|
||||
|
||||
},
|
||||
|
||||
});
|
||||
</script>
|
||||
|
||||
<script type="text/x-red" data-template-name="RAG">
|
||||
|
||||
<div class="form-row">
|
||||
<label for="node-input-name">Name</label>
|
||||
<input type="text" id="node-input-name" placeholder="Name" />
|
||||
</div>
|
||||
|
||||
<div class="form-row ">
|
||||
<label for="node-input-url"><i class="fa"></i>url</label>
|
||||
<input type="text" id="node-input-url" />
|
||||
</div>
|
||||
|
||||
<div class="form-row ">
|
||||
<label for="node-input-data_name"><i class="fa"></i>data_name</label>
|
||||
<input type="text" id="node-input-data_name" />
|
||||
</div>
|
||||
|
||||
<div class="form-row ">
|
||||
<label for="node-input-huggingface_token"><i class="fa"></i>huggingface_token</label>
|
||||
<input type="text" id="node-input-huggingface_token" />
|
||||
</div>
|
||||
|
||||
<div class="form-row ">
|
||||
<label for="node-input-embedding_model"><i class="fa"></i>embedding_model</label>
|
||||
<input type="text" id="node-input-embedding_model" />
|
||||
</div>
|
||||
|
||||
<div class="form-row ">
|
||||
<label for="node-input-LLM_model"><i class="fa"></i>LLM_model</label>
|
||||
<input type="text" id="node-input-LLM_model" />
|
||||
</div>
|
||||
<div class="form-row ">
|
||||
<label for="node-input-Pinecone_token"><i class="fa"></i>Pinecone_token</label>
|
||||
<input type="text" id="node-input-Pinecone_token" />
|
||||
</div>
|
||||
<div class="form-row ">
|
||||
<label for="node-input-Pinecone_index"><i class="fa"></i>Pinecone_index</label>
|
||||
<input type="text" id="node-input-Pinecone_index" />
|
||||
</div>
|
||||
|
||||
<div class="form-row ">
|
||||
<label for="node-input-host"><i class="fa"></i>host</label>
|
||||
<input type="text" id="node-input-host" />
|
||||
</div>
|
||||
|
||||
<div class="form-row ">
|
||||
<label for="node-input-username"><i class="fa"></i>username</label>
|
||||
<input type="text" id="node-input-username" />
|
||||
</div>
|
||||
|
||||
<div class="form-row ">
|
||||
<label for="node-input-userpassword"><i class="fa"></i>userpassword</label>
|
||||
<input type="text" id="node-input-userpassword" />
|
||||
</div>
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
120
RAG-pipeline-with-nodered/RAG_with_node-red/example/main/node_modules/autoAI pipeline/RAG/RAG.js
generated
vendored
Normal file
120
RAG-pipeline-with-nodered/RAG_with_node-red/example/main/node_modules/autoAI pipeline/RAG/RAG.js
generated
vendored
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
const util = require('util');
|
||||
const snippets = require('../snippets');
|
||||
const fs = require("fs");
|
||||
const spawn = require('child_process').spawn;
|
||||
|
||||
class PythonshellNode {
|
||||
constructor(config) {
|
||||
this.spawn = spawn;
|
||||
this.onStatus = ()=>{};
|
||||
}
|
||||
onInput(msg, node, out, err) {
|
||||
this.pyfile = msg.path;
|
||||
var spawnCmd = "python3";
|
||||
this.py = this.spawn(spawnCmd, ['-u', this.pyfile], {
|
||||
cwd: this.pyfile.substring(0, this.pyfile.lastIndexOf('/'))
|
||||
});
|
||||
this.onStatus({fill:"green",shape:"dot",text:"Standby"});
|
||||
|
||||
var py = this.py;
|
||||
var dataString = '';
|
||||
var errString = '';
|
||||
|
||||
py.stdout.on('data', data => {
|
||||
clearTimeout(this.standbyTimer)
|
||||
this.onStatus({fill:"green",shape:"dot",text:"Processing data"});
|
||||
let dataStr = data.toString();
|
||||
let lines = dataStr.split('\n');
|
||||
for (let line of lines) {
|
||||
if (line.startsWith("COLUMN_NAMES:")) {
|
||||
msg.col_names = line.split("COLUMN_NAMES:")[1].trim();
|
||||
}else if (line.startsWith("Classnames:")){
|
||||
msg.classnames = line.split("Classnames:")[1].trim();
|
||||
}else if (line.trim() !== "") {
|
||||
dataString += line + "\n";
|
||||
}
|
||||
}
|
||||
if (dataString != ""){
|
||||
node.warn(dataString.trim());
|
||||
}
|
||||
dataString = '';
|
||||
this.standbyTimer = setTimeout(()=>{
|
||||
this.onStatus({fill:"green",shape:"dot",text:"Standby"})
|
||||
}, 2000)
|
||||
});
|
||||
|
||||
py.stderr.on('data', data => {
|
||||
errString += String(data);
|
||||
this.onStatus({fill:"red",shape:"dot",text:"Error: " + errString});
|
||||
});
|
||||
py.stderr.on('error', console.log)
|
||||
py.stdout.on('error', console.log)
|
||||
py.stdin.on('error', console.log)
|
||||
py.on('error', console.log)
|
||||
|
||||
py.on('close', code => {
|
||||
if (code) {
|
||||
err('exit code: ' + code + ', ' + errString);
|
||||
this.onStatus({fill:"red",shape:"dot",text:"Exited: " + code})
|
||||
} else {
|
||||
out(msg);
|
||||
this.onStatus({fill:"green",shape:"dot",text:"Done"})
|
||||
}
|
||||
this.py = null;
|
||||
setTimeout(()=>{
|
||||
this.onStatus({})
|
||||
}, 2000)
|
||||
});
|
||||
};
|
||||
|
||||
onClose() {
|
||||
if (this.py) {
|
||||
this.py.kill();
|
||||
this.py = null;
|
||||
}
|
||||
}
|
||||
|
||||
setStatusCallback(callback) {
|
||||
this.onStatus = callback;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = function(RED) {
|
||||
function RAG(config) {
|
||||
RED.nodes.createNode(this,config);
|
||||
var node = this;
|
||||
var pyNode = new PythonshellNode(config);
|
||||
pyNode.setStatusCallback(node.status.bind(node));
|
||||
var script = '';
|
||||
node.on('input', function(msg) {
|
||||
|
||||
msg.path= "/data/main/py/RAG.py"
|
||||
|
||||
script = util.format(snippets.RAG, config.url, config.data_name, config.huggingface_token, config.embedding_model,
|
||||
config.LLM_model, config.Pinecone_token, config.Pinecone_index, config.host, config.username, config.userpassword)
|
||||
fs.writeFile(msg.path, script, function (err) {
|
||||
if (err) {
|
||||
node.error('Failed to write to file: ' + err.message);
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
pyNode.onInput(msg, node, function(result) {
|
||||
|
||||
msg.payload = config.client_num;
|
||||
node.send(msg);
|
||||
}, function(error) {
|
||||
node.error(error);
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
node.on('close', ()=>pyNode.onClose());
|
||||
}
|
||||
RED.nodes.registerType("RAG", RAG);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
19
RAG-pipeline-with-nodered/RAG_with_node-red/example/main/node_modules/autoAI pipeline/package.json
generated
vendored
Normal file
19
RAG-pipeline-with-nodered/RAG_with_node-red/example/main/node_modules/autoAI pipeline/package.json
generated
vendored
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"name": "node-red-contrib-autoai-pipeline",
|
||||
"version": "0.0.1",
|
||||
"description": "A set of Node-RED node to run pipeline on kubeflow",
|
||||
"scripts": {
|
||||
"test": "./node_modules/mocha/bin/mocha"
|
||||
},
|
||||
"license": "ISC",
|
||||
"keywords": [ "node-red" ],
|
||||
"node-red": {
|
||||
"nodes": {
|
||||
|
||||
"fl": "RAG/RAG.js"
|
||||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"mocha": "^5.0.4"
|
||||
}
|
||||
}
|
||||
342
RAG-pipeline-with-nodered/RAG_with_node-red/example/main/node_modules/autoAI pipeline/snippets.js
generated
vendored
Normal file
342
RAG-pipeline-with-nodered/RAG_with_node-red/example/main/node_modules/autoAI pipeline/snippets.js
generated
vendored
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
'use strict';
|
||||
const RAG =
|
||||
`
|
||||
import kfp
|
||||
from kfp import dsl
|
||||
from kfp.components import func_to_container_op
|
||||
from typing import NamedTuple
|
||||
|
||||
def RAG()-> NamedTuple('Outputs', [("output",str)]):
|
||||
import torch
|
||||
import requests
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
from langchain_community.vectorstores import Pinecone
|
||||
from langchain_community.document_loaders import PyPDFLoader, OnlinePDFLoader
|
||||
import os
|
||||
import sys
|
||||
from pinecone import Pinecone
|
||||
from langchain_pinecone import PineconeVectorStore
|
||||
from langchain.embeddings import HuggingFaceEmbeddings
|
||||
|
||||
import pinecone
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||||
from transformers import pipeline
|
||||
from langchain import HuggingFacePipeline, PromptTemplate
|
||||
from langchain.chains import RetrievalQA
|
||||
from huggingface_hub import login
|
||||
|
||||
|
||||
url = '%s'#Ex: "https://arxiv.org/pdf/2207.02696.pdf", a yolov7 pdf
|
||||
data_name = '%s'#Ex: "/yolov7.pdf"
|
||||
|
||||
huggingface_token = '%s'
|
||||
embedding_model = '%s'#Ex: "sentence-transformers/all-MiniLM-L6-v2"
|
||||
LLM_model = '%s'#Ex: "openai-community/gpt2" or "meta-llama/Llama-2-7b-chat-hf"
|
||||
Pinecone_token = '%s'
|
||||
Pinecone_index = '%s'
|
||||
|
||||
|
||||
r = requests.get(url, stream=True)
|
||||
with open(data_name, 'wb') as fd:
|
||||
fd.write(r.content)
|
||||
loader = PyPDFLoader(data_name)
|
||||
data = loader.load()
|
||||
|
||||
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=20)
|
||||
docs = text_splitter.split_documents(data)
|
||||
|
||||
embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
|
||||
PINE_KEY = os.environ.get("PINECONE_API_KEY", Pinecone_token)
|
||||
pc = Pinecone(api_key=Pinecone_token)
|
||||
os.environ["PINECONE_API_KEY"] = Pinecone_token
|
||||
index = pc.Index(Pinecone_index)
|
||||
index_name = Pinecone_index
|
||||
docsearch = PineconeVectorStore.from_documents(docs, embedding=embeddings, index_name=index_name)
|
||||
|
||||
|
||||
login(huggingface_token)
|
||||
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(LLM_model)
|
||||
model = AutoModelForCausalLM.from_pretrained(LLM_model,
|
||||
device_map='auto',
|
||||
torch_dtype=torch.float16,
|
||||
use_auth_token=True,
|
||||
load_in_8bit=True # If you don't use GPU, comment this parameter
|
||||
)
|
||||
model = pipeline("text-generation",
|
||||
model=model,
|
||||
tokenizer= tokenizer,
|
||||
torch_dtype=torch.bfloat16,
|
||||
device_map="auto",
|
||||
max_new_tokens = 512,
|
||||
do_sample=True,
|
||||
top_k=30,
|
||||
num_return_sequences=3,
|
||||
eos_token_id=tokenizer.eos_token_id
|
||||
)
|
||||
|
||||
llm=HuggingFacePipeline(pipeline=model, model_kwargs={'temperature':0.1})
|
||||
|
||||
SYSTEM_PROMPT = """Use the following pieces of context to answer the question at the end.
|
||||
If you don't know the answer, just say that you don't know, don't try to make up an answer."""
|
||||
B_INST, E_INST = "[INST]", "[/INST]"
|
||||
B_SYS, E_SYS = "<<SYS>>\\n", "\\n<</SYS>>\\n\\n"
|
||||
|
||||
SYSTEM_PROMPT = B_SYS + SYSTEM_PROMPT + E_SYS
|
||||
instruction = """
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
"""
|
||||
template = B_INST + SYSTEM_PROMPT + instruction + E_INST
|
||||
|
||||
prompt = PromptTemplate(template=template, input_variables=["context", "question"])
|
||||
|
||||
qa_chain = RetrievalQA.from_chain_type(
|
||||
llm=llm,
|
||||
chain_type="stuff",
|
||||
retriever=docsearch.as_retriever(search_kwargs={"k":3}),
|
||||
return_source_documents=True,
|
||||
chain_type_kwargs={"prompt": prompt},)
|
||||
|
||||
question = questions
|
||||
result = qa_chain(question)
|
||||
|
||||
print(result['result'])
|
||||
|
||||
output = result['result']
|
||||
return(output)
|
||||
|
||||
RAG_op = func_to_container_op(RAG,packages_to_install = [
|
||||
'requests',
|
||||
'langchain',
|
||||
'langchain-community',
|
||||
'pypdf',
|
||||
'pinecone-client',
|
||||
'langchain_pinecone',
|
||||
'sentence_transformers',
|
||||
'googletrans==3.1.0a0',
|
||||
'transformers',
|
||||
'torch',
|
||||
'bitsandbytes',
|
||||
'accelerate',
|
||||
'urllib3==1.26.15'
|
||||
])
|
||||
|
||||
@dsl.pipeline()
|
||||
def rag_pipeline():
|
||||
rag_task = RAG_op().set_gpu_limit(1)
|
||||
|
||||
from kfp import compiler
|
||||
|
||||
compiler.Compiler().compile(rag_pipeline, 'RAG_pipeline.yaml')
|
||||
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
import string
|
||||
import random
|
||||
import json
|
||||
from kfp import dsl
|
||||
from kfp.components import func_to_container_op, OutputPath
|
||||
from kfp_server_api.rest import ApiException
|
||||
from pprint import pprint
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = "%s"
|
||||
username = "%s"
|
||||
password = "%s"
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
def random_suffix() :
|
||||
return ''.join(random.choices(string.ascii_lowercase + string.digits, k=10))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the Experiment API class
|
||||
experiment_api_instance = kfp_server_api.ExperimentServiceApi(api_client)
|
||||
name="experiment-" + random_suffix()
|
||||
description="This is a experiment for RAG."
|
||||
resource_reference_key_id = namespaces[0]
|
||||
resource_references=[kfp_server_api.models.ApiResourceReference(
|
||||
key=kfp_server_api.models.ApiResourceKey(
|
||||
type=kfp_server_api.models.ApiResourceType.NAMESPACE,
|
||||
id=resource_reference_key_id
|
||||
),
|
||||
relationship=kfp_server_api.models.ApiRelationship.OWNER
|
||||
)]
|
||||
body = kfp_server_api.ApiExperiment(name=name, description=description, resource_references=resource_references) # ApiExperiment | The experiment to be created.
|
||||
try:
|
||||
# Creates a new experiment.
|
||||
experiment_api_response = experiment_api_instance.create_experiment(body)
|
||||
experiment_id = experiment_api_response.id # str | The ID of the run to be retrieved.
|
||||
except ApiException as e:
|
||||
print("Exception when calling ExperimentServiceApi->create_experiment: %s\\n" % e)
|
||||
|
||||
# Create an instance of the pipeline API class
|
||||
api_instance = kfp_server_api.PipelineUploadServiceApi(api_client)
|
||||
uploadfile="RAG_pipeline.yaml"
|
||||
name='pipeline-' + random_suffix()
|
||||
description="This is a RAG pipline."
|
||||
try:
|
||||
pipeline_api_response = api_instance.upload_pipeline(uploadfile, name=name, description=description)
|
||||
pipeline_id = pipeline_api_response.id # str | The ID of the run to be retrieved.
|
||||
except ApiException as e:
|
||||
print("Exception when calling PipelineUploadServiceApi->upload_pipeline: %s\\n" % e)
|
||||
|
||||
# Create an instance of the run API class
|
||||
run_api_instance = kfp_server_api.RunServiceApi(api_client)
|
||||
display_name = 'RAG' + random_suffix()
|
||||
description = "This is a RAG run."
|
||||
pipeline_spec = kfp_server_api.ApiPipelineSpec(pipeline_id=pipeline_id)
|
||||
resource_reference_key_id = namespaces[0]
|
||||
resource_references=[kfp_server_api.models.ApiResourceReference(
|
||||
key=kfp_server_api.models.ApiResourceKey(id=experiment_id, type=kfp_server_api.models.ApiResourceType.EXPERIMENT),
|
||||
relationship=kfp_server_api.models.ApiRelationship.OWNER )]
|
||||
body = kfp_server_api.ApiRun(name=display_name, description=description, pipeline_spec=pipeline_spec, resource_references=resource_references) # ApiRun |
|
||||
try:
|
||||
# Creates a new run.
|
||||
run_api_response = run_api_instance.create_run(body)
|
||||
run_id = run_api_response.run.id # str | The ID of the run to be retrieved.
|
||||
except ApiException as e:
|
||||
print("Exception when calling RunServiceApi->create_run: %s\\n" % e)
|
||||
|
||||
Completed_flag = False
|
||||
polling_interval = 10 # Time in seconds between polls
|
||||
|
||||
|
||||
|
||||
|
||||
while not Completed_flag:
|
||||
try:
|
||||
time.sleep(1)
|
||||
# Finds a specific run by ID.
|
||||
api_instance = run_api_instance.get_run(run_id)
|
||||
output = api_instance.pipeline_runtime.workflow_manifest
|
||||
output = json.loads(output)
|
||||
#print(output)
|
||||
|
||||
try:
|
||||
nodes = output['status']['nodes']
|
||||
conditions = output['status']['conditions'] # Comfirm completion.
|
||||
|
||||
except KeyError:
|
||||
nodes = {}
|
||||
conditions = []
|
||||
|
||||
output_value = None
|
||||
Completed_flag = conditions[1]['status'] if len(conditions) > 1 else False
|
||||
|
||||
'''''
|
||||
def find_all_keys(node):
|
||||
if isinstance(node, dict):
|
||||
for key in node.keys():
|
||||
print("Key:", key)
|
||||
find_all_keys(node[key])
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
find_all_keys(item)
|
||||
|
||||
# Call the function with your JSON data
|
||||
find_all_keys(output)
|
||||
'''''
|
||||
|
||||
except ApiException as e:
|
||||
print("Exception when calling RunServiceApi->get_run: %s\\n" % e)
|
||||
break
|
||||
|
||||
if not Completed_flag:
|
||||
print("Pipeline is still running. Waiting...")
|
||||
time.sleep(polling_interval-1)
|
||||
|
||||
found_final_pvc_name = False # Add a variable to track if the PVC name has been found
|
||||
|
||||
def find_final_pvc_name(node):
|
||||
global found_final_pvc_name # Declare the variable as global
|
||||
|
||||
if not found_final_pvc_name: # If the PVC name has not been found yet
|
||||
if isinstance(node, dict):
|
||||
if 'parameters' in node:
|
||||
parameters = node['parameters']
|
||||
for parameter in parameters:
|
||||
if 'name' in parameter and parameter['name'] == 'mypvc-name':
|
||||
value = parameter.get('value')
|
||||
if value and not value.startswith('{{') and not value.endswith('}}'):
|
||||
found_final_pvc_name = True # Set to True after finding the PVC name
|
||||
print("mypvc-name:", value)
|
||||
return value
|
||||
for key, value in node.items():
|
||||
result = find_final_pvc_name(value)
|
||||
if result:
|
||||
return result
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
result = find_final_pvc_name(item)
|
||||
if result:
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
find_final_pvc_name(output) # Call the function to find final_pvc_name
|
||||
|
||||
|
||||
found_model_func_accuracy = False
|
||||
|
||||
def find_model_func_accuracy(node):
|
||||
global found_model_func_accuracy # Declare the variable as global
|
||||
|
||||
if not found_model_func_accuracy: # If the model-func-accuracy has not been found yet
|
||||
if isinstance(node, dict):
|
||||
if 'parameters' in node:
|
||||
parameters = node['parameters']
|
||||
for parameter in parameters:
|
||||
if 'name' in parameter and parameter['name'] == 'model-func-accuracy':
|
||||
value = parameter.get('value')
|
||||
if value and not value.startswith('{{') and not value.endswith('}}'):
|
||||
found_model_func_accuracy = True # Set to True after finding model-func-accuracy
|
||||
print("Model Accuracy:", value)
|
||||
return value
|
||||
|
||||
for key, value in node.items():
|
||||
result = find_model_func_accuracy(value)
|
||||
if result:
|
||||
return result
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
result = find_model_func_accuracy(item)
|
||||
if result:
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
find_model_func_accuracy(output)
|
||||
`;
|
||||
|
||||
|
||||
|
||||
|
||||
module.exports = {
|
||||
RAG
|
||||
};
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"name": "node-red-project",
|
||||
"description": "A Node-RED Project",
|
||||
"version": "0.0.1",
|
||||
"private": true
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
from __future__ import print_function
|
||||
import string
|
||||
import random
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
import kfp
|
||||
import json
|
||||
from kfp_server_api.rest import ApiException
|
||||
from pprint import pprint
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the API class
|
||||
api_instance = kfp_server_api.ExperimentServiceApi(api_client)
|
||||
name="<change yours>" # str | The ID of the name to be create.
|
||||
description='<change yours>' # str | The description experiment.
|
||||
resource_reference_key_id = namespaces[0]
|
||||
resource_references=[kfp_server_api.models.ApiResourceReference(
|
||||
key=kfp_server_api.models.ApiResourceKey(
|
||||
type=kfp_server_api.models.ApiResourceType.NAMESPACE,
|
||||
id=resource_reference_key_id
|
||||
),
|
||||
relationship=kfp_server_api.models.ApiRelationship.OWNER
|
||||
)]
|
||||
body = kfp_server_api.ApiExperiment(name=name, description=description, resource_references=resource_references) # ApiExperiment | The experiment to be created.
|
||||
try:
|
||||
# Creates a new experiment.
|
||||
api_response = api_instance.create_experiment(body)
|
||||
pprint(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling ExperimentServiceApi->create_experiment: %s\n" % e)
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
from __future__ import print_function
|
||||
import string
|
||||
import random
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
import kfp
|
||||
import json
|
||||
from pprint import pprint
|
||||
from kfp_server_api.rest import ApiException
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the API class
|
||||
api_instance = kfp_server_api.RunServiceApi(api_client)
|
||||
pipeline_id = '<change yours>' # str | The ID of the pipeline.
|
||||
experiment_id = '<change yours>' # str | The ID of the experiment.
|
||||
display_name = '<change yours>' # str | The name of the run to be create.
|
||||
description = '<change yours>' # str | The description of run.
|
||||
pipeline_spec = kfp_server_api.ApiPipelineSpec(pipeline_id=pipeline_id)
|
||||
resource_reference_key_id = namespaces[0]
|
||||
resource_references=[kfp_server_api.models.ApiResourceReference(
|
||||
key=kfp_server_api.models.ApiResourceKey(id=experiment_id, type=kfp_server_api.models.ApiResourceType.EXPERIMENT),
|
||||
relationship=kfp_server_api.models.ApiRelationship.OWNER )]
|
||||
body = kfp_server_api.ApiRun(name=display_name, description=description, pipeline_spec=pipeline_spec, resource_references=resource_references) # ApiRun |
|
||||
|
||||
try:
|
||||
# Creates a new run.
|
||||
api_response = api_instance.create_run(body)
|
||||
pprint(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling RunServiceApi->create_run: %s\n" % e)
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
from __future__ import print_function
|
||||
import string
|
||||
import random
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
import kfp
|
||||
import json
|
||||
from kfp_server_api.rest import ApiException
|
||||
from pprint import pprint
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the API class
|
||||
api_instance = kfp_server_api.ExperimentServiceApi(api_client)
|
||||
experiment_id = '<change yours>' # str | The ID of the experiment to be deleted.
|
||||
|
||||
try:
|
||||
# Deletes an experiment without deleting the experiment's runs and recurring runs.
|
||||
# To avoid unexpected behaviors, delete an experiment's runs and recurring runs before deleting the experiment.
|
||||
api_response = api_instance.delete_experiment(experiment_id)
|
||||
pprint(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling ExperimentServiceApi->delete_experiment: %s\n" % e)
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
from __future__ import print_function
|
||||
import string
|
||||
import random
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
import kfp
|
||||
import json
|
||||
from kfp_server_api.rest import ApiException
|
||||
from pprint import pprint
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the API class
|
||||
api_instance = kfp_server_api.PipelineServiceApi(api_client)
|
||||
id = '<change yours>' # str | The ID of the pipeline to be deleted.
|
||||
|
||||
try:
|
||||
# Deletes a pipeline and its pipeline versions.
|
||||
api_response = api_instance.delete_pipeline(id)
|
||||
pprint(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling PipelineServiceApi->delete_pipeline: %s\n" % e)
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
from __future__ import print_function
|
||||
import string
|
||||
import random
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
import kfp
|
||||
import json
|
||||
from kfp_server_api.rest import ApiException
|
||||
from pprint import pprint
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the API class
|
||||
api_instance = kfp_server_api.RunServiceApi(api_client)
|
||||
id = '<change yours>' # str | The ID of the run to be deleted.
|
||||
|
||||
try:
|
||||
# Deletes a run.
|
||||
api_response = api_instance.delete_run(id)
|
||||
pprint(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling RunServiceApi->delete_run: %s\n" % e)
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
from __future__ import print_function
|
||||
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
from kfp_server_api.rest import ApiException
|
||||
from pprint import pprint
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
import json
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the API class
|
||||
api_instance = kfp_server_api.RunServiceApi(api_client)
|
||||
run_id = '<change yours>' # str | The ID of the run to be retrieved.
|
||||
|
||||
try:
|
||||
# Finds a specific run by ID.
|
||||
api_response = api_instance.get_run(run_id)
|
||||
output = api_response.pipeline_runtime.workflow_manifest
|
||||
output = json.loads(output)
|
||||
nodes = output['status']['nodes']
|
||||
conditions = output['status']['conditions'] # Comfirm completion.
|
||||
output_value = None
|
||||
|
||||
for node_id, node in nodes.items():
|
||||
if 'inputs' in node and 'parameters' in node['inputs']:
|
||||
for parameter in node['inputs']['parameters']:
|
||||
if parameter['name'] == 'decision-tree-classifier-Accuracy':
|
||||
output_value = parameter['value']
|
||||
break
|
||||
|
||||
if output_value is not None:
|
||||
print(f"Decision Tree Classifier Accuracy: {output_value}")
|
||||
else:
|
||||
print("Parameter not found.")
|
||||
except ApiException as e:
|
||||
print("Exception when calling RunServiceApi->get_run: %s\n" % e)
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
import re
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
# NOTE: the following code is referred from https://github.com/kubeflow/website/issues/2916
|
||||
def get_istio_auth_session(url: str, username: str, password: str) -> dict:
|
||||
"""
|
||||
Determine if the specified URL is secured by Dex and try to obtain a session cookie.
|
||||
WARNING: only Dex `staticPasswords` and `LDAP` authentication are currently supported
|
||||
(we default default to using `staticPasswords` if both are enabled)
|
||||
|
||||
:param url: Kubeflow server URL, including protocol
|
||||
:param username: Dex `staticPasswords` or `LDAP` username
|
||||
:param password: Dex `staticPasswords` or `LDAP` password
|
||||
:return: auth session information
|
||||
"""
|
||||
# define the default return object
|
||||
auth_session = {
|
||||
"endpoint_url": url, # KF endpoint URL
|
||||
"redirect_url": None, # KF redirect URL, if applicable
|
||||
"dex_login_url": None, # Dex login URL (for POST of credentials)
|
||||
"is_secured": None, # True if KF endpoint is secured
|
||||
"session_cookie": None # Resulting session cookies in the form "key1=value1; key2=value2"
|
||||
}
|
||||
|
||||
# use a persistent session (for cookies)
|
||||
with requests.Session() as s:
|
||||
|
||||
################
|
||||
# Determine if Endpoint is Secured
|
||||
################
|
||||
resp = s.get(url, allow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {url}"
|
||||
)
|
||||
|
||||
auth_session["redirect_url"] = resp.url
|
||||
|
||||
# if we were NOT redirected, then the endpoint is UNSECURED
|
||||
if len(resp.history) == 0:
|
||||
auth_session["is_secured"] = False
|
||||
return auth_session
|
||||
else:
|
||||
auth_session["is_secured"] = True
|
||||
|
||||
################
|
||||
# Get Dex Login URL
|
||||
################
|
||||
redirect_url_obj = urlsplit(auth_session["redirect_url"])
|
||||
|
||||
# if we are at `/auth?=xxxx` path, we need to select an auth type
|
||||
if re.search(r"/auth$", redirect_url_obj.path):
|
||||
# default to "staticPasswords" auth type
|
||||
redirect_url_obj = redirect_url_obj._replace(
|
||||
path=re.sub(r"/auth$", "/auth/local", redirect_url_obj.path)
|
||||
)
|
||||
|
||||
# if we are at `/auth/xxxx/login` path, then no further action is needed (we can use it for login POST)
|
||||
if re.search(r"/auth/.*/login$", redirect_url_obj.path):
|
||||
auth_session["dex_login_url"] = redirect_url_obj.geturl()
|
||||
|
||||
# else, we need to be redirected to the actual login page
|
||||
else:
|
||||
# this GET should redirect us to the `/auth/xxxx/login` path
|
||||
resp = s.get(redirect_url_obj.geturl(), allow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {redirect_url_obj.geturl()}"
|
||||
)
|
||||
|
||||
# set the login url
|
||||
auth_session["dex_login_url"] = resp.url
|
||||
|
||||
################
|
||||
# Attempt Dex Login
|
||||
################
|
||||
resp = s.post(
|
||||
auth_session["dex_login_url"],
|
||||
data={"login": username, "password": password},
|
||||
allow_redirects=True
|
||||
)
|
||||
if len(resp.history) == 0:
|
||||
raise RuntimeError(
|
||||
f"Login credentials were probably invalid - "
|
||||
f"No redirect after POST to: {auth_session['dex_login_url']}"
|
||||
)
|
||||
|
||||
# store the session cookies in a "key1=value1; key2=value2" string
|
||||
auth_session["session_cookie"] = "; ".join([f"{c.name}={c.value}" for c in s.cookies])
|
||||
|
||||
return auth_session
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import os
|
||||
import requests
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
def retrieve_namespaces(host: str, auth_session: TypedDict) -> str:
|
||||
workgroup_endpoint = os.path.join(host, "api/workgroup/env-info")
|
||||
|
||||
cookies = {}
|
||||
cookie_tokens = auth_session["session_cookie"].split("=")
|
||||
print(cookie_tokens[0])
|
||||
cookies[cookie_tokens[0]]=cookie_tokens[1]
|
||||
resp = requests.get(workgroup_endpoint, cookies=cookies)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {workgroup_endpoint}"
|
||||
)
|
||||
return [ns["namespace"] for ns in resp.json()["namespaces"] if
|
||||
ns["role"]=="owner"]
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
from __future__ import print_function
|
||||
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
from kfp_server_api.rest import ApiException
|
||||
from pprint import pprint
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
print("available namespace: {}".format(namespaces))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the API class
|
||||
api_instance = kfp_server_api.ExperimentServiceApi(api_client)
|
||||
resource_reference_key_type = "NAMESPACE"
|
||||
resource_reference_key_id = namespaces[0]
|
||||
list_experiment_response = api_instance.list_experiment(resource_reference_key_type=resource_reference_key_type, resource_reference_key_id=resource_reference_key_id)
|
||||
for experiment in list_experiment_response.experiments:
|
||||
pprint(experiment)
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
from __future__ import print_function
|
||||
import string
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
import kfp
|
||||
import json
|
||||
from pprint import pprint
|
||||
from kfp_server_api.rest import ApiException
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
api_instance = kfp_server_api.PipelineServiceApi(api_client)
|
||||
list_pipeline_response = api_instance.list_pipelines()
|
||||
for pipelines in list_pipeline_response.pipelines:
|
||||
print(pipelines)
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
from __future__ import print_function
|
||||
import string
|
||||
import random
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
from kfp_server_api.rest import ApiException
|
||||
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
|
||||
host = os.getenv("KUBEFLOW_HOST")
|
||||
username = os.getenv("KUBEFLOW_USERNAME")
|
||||
password = os.getenv("KUBEFLOW_PASSWORD")
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
print("available namespace: {}".format(namespaces))
|
||||
|
||||
def random_suffix() -> string:
|
||||
return ''.join(random.choices(string.ascii_lowercase + string.digits, k=10))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the API class
|
||||
api_instance = kfp_server_api.PipelineUploadServiceApi(api_client)
|
||||
uploadfile='<change yours>' # The yaml file in your local path.
|
||||
name='pipeline-' + random_suffix()
|
||||
description='<change yours>' # str | The description of pipeline.
|
||||
try:
|
||||
api_response = api_instance.upload_pipeline(uploadfile, name=name, description=description)
|
||||
print(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling PipelineUploadServiceApi->upload_pipeline: %s\n" % e)
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
|
||||
from __future__ import print_function
|
||||
|
||||
import re
|
||||
import kfp
|
||||
import time
|
||||
import kfp_server_api
|
||||
import os
|
||||
import requests
|
||||
import string
|
||||
import random
|
||||
import json
|
||||
from kfp import dsl
|
||||
from kfp import components
|
||||
from kfp.components import func_to_container_op, OutputPath
|
||||
from kfp_server_api.rest import ApiException
|
||||
from pprint import pprint
|
||||
from kfp_login import get_istio_auth_session
|
||||
from kfp_namespace import retrieve_namespaces
|
||||
|
||||
host = ""
|
||||
username = ""
|
||||
password = ""
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
namespaces = str(namespaces)
|
||||
namespaces = namespaces[2:-2]
|
||||
#print(namespaces)
|
||||
|
||||
|
||||
kfserving_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kserve/component.yaml')
|
||||
@dsl.pipeline(
|
||||
name='KFServing pipeline',
|
||||
description='A pipeline for KFServing with PVC.'
|
||||
)
|
||||
def kfservingPipeline(
|
||||
action='apply',
|
||||
namespace=namespaces,
|
||||
|
||||
pvc_name='undefined', # change pvc_name
|
||||
model_name='model01'): # change model_name
|
||||
|
||||
# specify the model dir located on pvc
|
||||
model_pvc_uri = 'pvc://{}/{}/'.format(pvc_name, model_name)
|
||||
|
||||
# create inference service resource named by model_name
|
||||
isvc_yaml = '''
|
||||
apiVersion: "serving.kserve.io/v1beta1"
|
||||
kind: "InferenceService"
|
||||
metadata:
|
||||
name: {}
|
||||
namespace: {}
|
||||
spec:
|
||||
predictor:
|
||||
sklearn:
|
||||
storageUri: {}
|
||||
resources:
|
||||
limits:
|
||||
cpu: "100m"
|
||||
requests:
|
||||
cpu: "100m"
|
||||
'''.format(model_name, namespace, model_pvc_uri)
|
||||
|
||||
|
||||
|
||||
kfserving = kfserving_op(
|
||||
action=action,
|
||||
inferenceservice_yaml=isvc_yaml,
|
||||
)
|
||||
kfserving.set_cpu_request("500m").set_cpu_limit("500m")
|
||||
|
||||
return([isvc_yaml])
|
||||
|
||||
|
||||
# Compile pipeline
|
||||
kfp.compiler.Compiler().compile(kfservingPipeline, 'sklearn-kserve.yaml')
|
||||
|
||||
host = "http://140.128.102.163:31740"
|
||||
username = "kubeflow02@gmail.com"
|
||||
password = "tkiizd"
|
||||
|
||||
auth_session = get_istio_auth_session(
|
||||
url=host,
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
|
||||
# The client must configure the authentication and authorization parameters
|
||||
# in accordance with the API server security policy.
|
||||
# Examples for each auth method are provided below, use the example that
|
||||
# satisfies your auth use case.
|
||||
|
||||
# Configure API key authorization: Bearer
|
||||
configuration = kfp_server_api.Configuration(
|
||||
host = os.path.join(host, "pipeline"),
|
||||
)
|
||||
configuration.debug = True
|
||||
namespaces = retrieve_namespaces(host, auth_session)
|
||||
#print("available namespace: {}".format(namespaces))
|
||||
|
||||
def random_suffix() :
|
||||
return ''.join(random.choices(string.ascii_lowercase + string.digits, k=10))
|
||||
|
||||
# Enter a context with an instance of the API client
|
||||
with kfp_server_api.ApiClient(configuration, cookie=auth_session["session_cookie"]) as api_client:
|
||||
# Create an instance of the Experiment API class
|
||||
experiment_api_instance = kfp_server_api.ExperimentServiceApi(api_client)
|
||||
name="experiment-" + random_suffix()
|
||||
description="This is a experiment for only_kfserving."
|
||||
resource_reference_key_id = namespaces[0]
|
||||
resource_references=[kfp_server_api.models.ApiResourceReference(
|
||||
key=kfp_server_api.models.ApiResourceKey(
|
||||
type=kfp_server_api.models.ApiResourceType.NAMESPACE,
|
||||
id=resource_reference_key_id
|
||||
),
|
||||
relationship=kfp_server_api.models.ApiRelationship.OWNER
|
||||
)]
|
||||
body = kfp_server_api.ApiExperiment(name=name, description=description, resource_references=resource_references) # ApiExperiment | The experiment to be created.
|
||||
try:
|
||||
# Creates a new experiment.
|
||||
experiment_api_response = experiment_api_instance.create_experiment(body)
|
||||
experiment_id = experiment_api_response.id # str | The ID of the run to be retrieved.
|
||||
except ApiException as e:
|
||||
print("Exception when calling ExperimentServiceApi->create_experiment: %s\n" % e)
|
||||
|
||||
# Create an instance of the pipeline API class
|
||||
api_instance = kfp_server_api.PipelineUploadServiceApi(api_client)
|
||||
uploadfile='sklearn-kserve.yaml'
|
||||
name='pipeline-' + random_suffix()
|
||||
description="This is a only_kfserving pipline."
|
||||
try:
|
||||
pipeline_api_response = api_instance.upload_pipeline(uploadfile, name=name, description=description)
|
||||
pipeline_id = pipeline_api_response.id # str | The ID of the run to be retrieved.
|
||||
except ApiException as e:
|
||||
print("Exception when calling PipelineUploadServiceApi->upload_pipeline: %s\n" % e)
|
||||
|
||||
# Create an instance of the run API class
|
||||
run_api_instance = kfp_server_api.RunServiceApi(api_client)
|
||||
display_name = 'run_only_kfserving' + random_suffix()
|
||||
description = "This is a only_kfserving run."
|
||||
pipeline_spec = kfp_server_api.ApiPipelineSpec(pipeline_id=pipeline_id)
|
||||
resource_reference_key_id = namespaces[0]
|
||||
resource_references=[kfp_server_api.models.ApiResourceReference(
|
||||
key=kfp_server_api.models.ApiResourceKey(id=experiment_id, type=kfp_server_api.models.ApiResourceType.EXPERIMENT),
|
||||
relationship=kfp_server_api.models.ApiRelationship.OWNER )]
|
||||
body = kfp_server_api.ApiRun(name=display_name, description=description, pipeline_spec=pipeline_spec, resource_references=resource_references) # ApiRun |
|
||||
try:
|
||||
# Creates a new run.
|
||||
run_api_response = run_api_instance.create_run(body)
|
||||
run_id = run_api_response.run.id # str | The ID of the run to be retrieved.
|
||||
except ApiException as e:
|
||||
print("Exception when calling RunServiceApi->create_run: %s\n" % e)
|
||||
|
||||
Completed_flag = False
|
||||
polling_interval = 10 # Time in seconds between polls
|
||||
|
||||
while not Completed_flag:
|
||||
try:
|
||||
time.sleep(1)
|
||||
# Finds a specific run by ID.
|
||||
api_instance = run_api_instance.get_run(run_id)
|
||||
output = api_instance.pipeline_runtime.workflow_manifest
|
||||
output = json.loads(output)
|
||||
|
||||
try:
|
||||
nodes = output['status']['nodes']
|
||||
conditions = output['status']['conditions'] # Confirm completion.
|
||||
|
||||
except KeyError:
|
||||
nodes = {}
|
||||
conditions = []
|
||||
|
||||
Completed_flag = conditions[1]['status'] if len(conditions) > 1 else False
|
||||
|
||||
except ApiException as e:
|
||||
print("Exception when calling RunServiceApi->get_run: %s\n" % e)
|
||||
break
|
||||
|
||||
if not Completed_flag:
|
||||
print("Pipeline is still running. Waiting...")
|
||||
time.sleep(polling_interval-1)
|
||||
else:
|
||||
print("Successful Deployment!")
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
import re
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
# NOTE: the following code is referred from https://github.com/kubeflow/website/issues/2916
|
||||
def get_istio_auth_session(url: str, username: str, password: str) -> dict:
|
||||
"""
|
||||
Determine if the specified URL is secured by Dex and try to obtain a session cookie.
|
||||
WARNING: only Dex `staticPasswords` and `LDAP` authentication are currently supported
|
||||
(we default default to using `staticPasswords` if both are enabled)
|
||||
|
||||
:param url: Kubeflow server URL, including protocol
|
||||
:param username: Dex `staticPasswords` or `LDAP` username
|
||||
:param password: Dex `staticPasswords` or `LDAP` password
|
||||
:return: auth session information
|
||||
"""
|
||||
# define the default return object
|
||||
auth_session = {
|
||||
"endpoint_url": url, # KF endpoint URL
|
||||
"redirect_url": None, # KF redirect URL, if applicable
|
||||
"dex_login_url": None, # Dex login URL (for POST of credentials)
|
||||
"is_secured": None, # True if KF endpoint is secured
|
||||
"session_cookie": None # Resulting session cookies in the form "key1=value1; key2=value2"
|
||||
}
|
||||
|
||||
# use a persistent session (for cookies)
|
||||
with requests.Session() as s:
|
||||
|
||||
################
|
||||
# Determine if Endpoint is Secured
|
||||
################
|
||||
resp = s.get(url, allow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {url}"
|
||||
)
|
||||
|
||||
auth_session["redirect_url"] = resp.url
|
||||
|
||||
# if we were NOT redirected, then the endpoint is UNSECURED
|
||||
if len(resp.history) == 0:
|
||||
auth_session["is_secured"] = False
|
||||
return auth_session
|
||||
else:
|
||||
auth_session["is_secured"] = True
|
||||
|
||||
################
|
||||
# Get Dex Login URL
|
||||
################
|
||||
redirect_url_obj = urlsplit(auth_session["redirect_url"])
|
||||
|
||||
# if we are at `/auth?=xxxx` path, we need to select an auth type
|
||||
if re.search(r"/auth$", redirect_url_obj.path):
|
||||
# default to "staticPasswords" auth type
|
||||
redirect_url_obj = redirect_url_obj._replace(
|
||||
path=re.sub(r"/auth$", "/auth/local", redirect_url_obj.path)
|
||||
)
|
||||
|
||||
# if we are at `/auth/xxxx/login` path, then no further action is needed (we can use it for login POST)
|
||||
if re.search(r"/auth/.*/login$", redirect_url_obj.path):
|
||||
auth_session["dex_login_url"] = redirect_url_obj.geturl()
|
||||
|
||||
# else, we need to be redirected to the actual login page
|
||||
else:
|
||||
# this GET should redirect us to the `/auth/xxxx/login` path
|
||||
resp = s.get(redirect_url_obj.geturl(), allow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {redirect_url_obj.geturl()}"
|
||||
)
|
||||
|
||||
# set the login url
|
||||
auth_session["dex_login_url"] = resp.url
|
||||
|
||||
################
|
||||
# Attempt Dex Login
|
||||
################
|
||||
resp = s.post(
|
||||
auth_session["dex_login_url"],
|
||||
data={"login": username, "password": password},
|
||||
allow_redirects=True
|
||||
)
|
||||
if len(resp.history) == 0:
|
||||
raise RuntimeError(
|
||||
f"Login credentials were probably invalid - "
|
||||
f"No redirect after POST to: {auth_session['dex_login_url']}"
|
||||
)
|
||||
|
||||
# store the session cookies in a "key1=value1; key2=value2" string
|
||||
auth_session["session_cookie"] = "; ".join([f"{c.name}={c.value}" for c in s.cookies])
|
||||
|
||||
return auth_session
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import os
|
||||
import requests
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
def retrieve_namespaces(host: str, auth_session: TypedDict) -> str:
|
||||
workgroup_endpoint = os.path.join(host, "api/workgroup/env-info")
|
||||
|
||||
cookies = {}
|
||||
cookie_tokens = auth_session["session_cookie"].split("=")
|
||||
print(cookie_tokens[0])
|
||||
cookies[cookie_tokens[0]]=cookie_tokens[1]
|
||||
resp = requests.get(workgroup_endpoint, cookies=cookies)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {workgroup_endpoint}"
|
||||
)
|
||||
return [ns["namespace"] for ns in resp.json()["namespaces"] if
|
||||
ns["role"]=="owner"]
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
def python3_version():
|
||||
return subprocess.check_call(["python3", "--version"])
|
||||
|
||||
def which(command):
|
||||
return subprocess.check_call(["which", command])
|
||||
|
||||
def pip3_install_requirements():
|
||||
return subprocess.check_call(["pip3", "install", "-r", "requirements.txt",
|
||||
"--user"])
|
||||
|
||||
def pip3_install_kfp_sever_api():
|
||||
return subprocess.check_call(["pip3", "install",
|
||||
"git+https://github.com/kubeflow/pipelines.git@1.8.19#subdirectory=backend/api/python_http_client",
|
||||
"--user"])
|
||||
|
||||
def pip3_install_kfp():
|
||||
return subprocess.check_call(["pip3", "install", "kfp==1.7", "--user"])
|
||||
|
||||
python3_version()
|
||||
pip3_install_requirements()
|
||||
#pip3_install_kfp_sever_api()
|
||||
pip3_install_kfp()
|
||||
|
||||
print("done")
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
import re
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
# NOTE: the following code is referred from https://github.com/kubeflow/website/issues/2916
|
||||
def get_istio_auth_session(url: str, username: str, password: str) -> dict:
|
||||
"""
|
||||
Determine if the specified URL is secured by Dex and try to obtain a session cookie.
|
||||
WARNING: only Dex `staticPasswords` and `LDAP` authentication are currently supported
|
||||
(we default default to using `staticPasswords` if both are enabled)
|
||||
|
||||
:param url: Kubeflow server URL, including protocol
|
||||
:param username: Dex `staticPasswords` or `LDAP` username
|
||||
:param password: Dex `staticPasswords` or `LDAP` password
|
||||
:return: auth session information
|
||||
"""
|
||||
# define the default return object
|
||||
auth_session = {
|
||||
"endpoint_url": url, # KF endpoint URL
|
||||
"redirect_url": None, # KF redirect URL, if applicable
|
||||
"dex_login_url": None, # Dex login URL (for POST of credentials)
|
||||
"is_secured": None, # True if KF endpoint is secured
|
||||
"session_cookie": None # Resulting session cookies in the form "key1=value1; key2=value2"
|
||||
}
|
||||
|
||||
# use a persistent session (for cookies)
|
||||
with requests.Session() as s:
|
||||
|
||||
################
|
||||
# Determine if Endpoint is Secured
|
||||
################
|
||||
resp = s.get(url, allow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {url}"
|
||||
)
|
||||
|
||||
auth_session["redirect_url"] = resp.url
|
||||
|
||||
# if we were NOT redirected, then the endpoint is UNSECURED
|
||||
if len(resp.history) == 0:
|
||||
auth_session["is_secured"] = False
|
||||
return auth_session
|
||||
else:
|
||||
auth_session["is_secured"] = True
|
||||
|
||||
################
|
||||
# Get Dex Login URL
|
||||
################
|
||||
redirect_url_obj = urlsplit(auth_session["redirect_url"])
|
||||
|
||||
# if we are at `/auth?=xxxx` path, we need to select an auth type
|
||||
if re.search(r"/auth$", redirect_url_obj.path):
|
||||
# default to "staticPasswords" auth type
|
||||
redirect_url_obj = redirect_url_obj._replace(
|
||||
path=re.sub(r"/auth$", "/auth/local", redirect_url_obj.path)
|
||||
)
|
||||
|
||||
# if we are at `/auth/xxxx/login` path, then no further action is needed (we can use it for login POST)
|
||||
if re.search(r"/auth/.*/login$", redirect_url_obj.path):
|
||||
auth_session["dex_login_url"] = redirect_url_obj.geturl()
|
||||
|
||||
# else, we need to be redirected to the actual login page
|
||||
else:
|
||||
# this GET should redirect us to the `/auth/xxxx/login` path
|
||||
resp = s.get(redirect_url_obj.geturl(), allow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {redirect_url_obj.geturl()}"
|
||||
)
|
||||
|
||||
# set the login url
|
||||
auth_session["dex_login_url"] = resp.url
|
||||
|
||||
################
|
||||
# Attempt Dex Login
|
||||
################
|
||||
resp = s.post(
|
||||
auth_session["dex_login_url"],
|
||||
data={"login": username, "password": password},
|
||||
allow_redirects=True
|
||||
)
|
||||
if len(resp.history) == 0:
|
||||
raise RuntimeError(
|
||||
f"Login credentials were probably invalid - "
|
||||
f"No redirect after POST to: {auth_session['dex_login_url']}"
|
||||
)
|
||||
|
||||
# store the session cookies in a "key1=value1; key2=value2" string
|
||||
auth_session["session_cookie"] = "; ".join([f"{c.name}={c.value}" for c in s.cookies])
|
||||
|
||||
return auth_session
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import os
|
||||
import requests
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
def retrieve_namespaces(host: str, auth_session: TypedDict) -> str:
|
||||
workgroup_endpoint = os.path.join(host, "api/workgroup/env-info")
|
||||
|
||||
cookies = {}
|
||||
cookie_tokens = auth_session["session_cookie"].split("=")
|
||||
print(cookie_tokens[0])
|
||||
cookies[cookie_tokens[0]]=cookie_tokens[1]
|
||||
resp = requests.get(workgroup_endpoint, cookies=cookies)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP status code '{resp.status_code}' for GET against: {workgroup_endpoint}"
|
||||
)
|
||||
return [ns["namespace"] for ns in resp.json()["namespaces"] if
|
||||
ns["role"]=="owner"]
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Workflow
|
||||
metadata:
|
||||
generateName: only-decision-tree-
|
||||
annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.20, pipelines.kubeflow.org/pipeline_compilation_time: '2023-04-28T21:42:00.682134',
|
||||
pipelines.kubeflow.org/pipeline_spec: '{"description": "Applies Decision Tree
|
||||
for classification problem.", "name": "only_decision_tree"}'}
|
||||
labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.20}
|
||||
spec:
|
||||
entrypoint: only-decision-tree
|
||||
templates:
|
||||
- name: decision-tree-classifier
|
||||
container:
|
||||
args: []
|
||||
command: [python, decision_tree.py, --data, /tmp/inputs/Data/data, --accuracy,
|
||||
/tmp/outputs/Accuracy/data]
|
||||
image: lightnighttw/kubeflow:decision_tree_v2
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/inputs/Data/data}
|
||||
outputs:
|
||||
parameters:
|
||||
- name: decision-tree-classifier-Accuracy
|
||||
valueFrom: {path: /tmp/outputs/Accuracy/data}
|
||||
artifacts:
|
||||
- {name: decision-tree-classifier-Accuracy, path: /tmp/outputs/Accuracy/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Trains
|
||||
a decision tree classifier", "implementation": {"container": {"command":
|
||||
["python", "decision_tree.py", "--data", {"inputPath": "Data"}, "--accuracy",
|
||||
{"outputPath": "Accuracy"}], "image": "lightnighttw/kubeflow:decision_tree_v2"}},
|
||||
"inputs": [{"description": "Path where data is stored.", "name": "Data",
|
||||
"type": "LocalPath"}], "name": "Decision Tree classifier", "outputs": [{"description":
|
||||
"Accuracy metric", "name": "Accuracy", "type": "Float"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
|
||||
"c5c232a9654213b3b222693949b71a5d561d04ed09543c20b6f8f2eab651ae0b", "url":
|
||||
"decision_tree/decision_tree.yaml"}'}
|
||||
- name: download-data-function
|
||||
container:
|
||||
args: []
|
||||
command: [python, download_data.py, --data, /tmp/outputs/Data/data]
|
||||
image: lightnighttw/kubeflow:download_data
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
outputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/outputs/Data/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Download
|
||||
toy data from sklearn datasets", "implementation": {"container": {"command":
|
||||
["python", "download_data.py", "--data", {"outputPath": "Data"}], "image":
|
||||
"lightnighttw/kubeflow:download_data"}}, "name": "Download Data Function",
|
||||
"outputs": [{"description": "Path where data will be stored.", "name": "Data",
|
||||
"type": "LocalPath"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
|
||||
"467750defdccfec51c3af2a7eb853f74235f5f97329006d72bf33ff6e15ed02d", "url":
|
||||
"download_data/download_data.yaml"}'}
|
||||
- name: only-decision-tree
|
||||
dag:
|
||||
tasks:
|
||||
- name: decision-tree-classifier
|
||||
template: decision-tree-classifier
|
||||
dependencies: [download-data-function]
|
||||
arguments:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, from: '{{tasks.download-data-function.outputs.artifacts.download-data-function-Data}}'}
|
||||
- {name: download-data-function, template: download-data-function}
|
||||
- name: show-results
|
||||
template: show-results
|
||||
dependencies: [decision-tree-classifier]
|
||||
arguments:
|
||||
parameters:
|
||||
- {name: decision-tree-classifier-Accuracy, value: '{{tasks.decision-tree-classifier.outputs.parameters.decision-tree-classifier-Accuracy}}'}
|
||||
- name: show-results
|
||||
container:
|
||||
args: [--decision-tree, '{{inputs.parameters.decision-tree-classifier-Accuracy}}']
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
program_path=$(mktemp)
|
||||
printf "%s" "$0" > "$program_path"
|
||||
python3 -u "$program_path" "$@"
|
||||
- |
|
||||
def show_results(decision_tree):
|
||||
# the results are shown.
|
||||
|
||||
print(f"Decision tree (accuracy): {decision_tree}")
|
||||
|
||||
import argparse
|
||||
_parser = argparse.ArgumentParser(prog='Show results', description='')
|
||||
_parser.add_argument("--decision-tree", dest="decision_tree", type=float, required=True, default=argparse.SUPPRESS)
|
||||
_parsed_args = vars(_parser.parse_args())
|
||||
|
||||
_outputs = show_results(**_parsed_args)
|
||||
image: python:3.7
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
parameters:
|
||||
- {name: decision-tree-classifier-Accuracy}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container":
|
||||
{"args": ["--decision-tree", {"inputValue": "decision_tree"}], "command":
|
||||
["sh", "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3
|
||||
-u \"$program_path\" \"$@\"\n", "def show_results(decision_tree):\n #
|
||||
the results are shown.\n\n print(f\"Decision tree (accuracy): {decision_tree}\")\n\nimport
|
||||
argparse\n_parser = argparse.ArgumentParser(prog=''Show results'', description='''')\n_parser.add_argument(\"--decision-tree\",
|
||||
dest=\"decision_tree\", type=float, required=True, default=argparse.SUPPRESS)\n_parsed_args
|
||||
= vars(_parser.parse_args())\n\n_outputs = show_results(**_parsed_args)\n"],
|
||||
"image": "python:3.7"}}, "inputs": [{"name": "decision_tree", "type": "Float"}],
|
||||
"name": "Show results"}', pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"decision_tree":
|
||||
"{{inputs.parameters.decision-tree-classifier-Accuracy}}"}'}
|
||||
arguments:
|
||||
parameters: []
|
||||
serviceAccountName: pipeline-runner
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Workflow
|
||||
metadata:
|
||||
generateName: only-logistic-regression-
|
||||
annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.20, pipelines.kubeflow.org/pipeline_compilation_time: '2023-04-28T21:47:07.033809',
|
||||
pipelines.kubeflow.org/pipeline_spec: '{"description": "Applies Logistic Regression
|
||||
for classification problem.", "name": "only_logistic_regression"}'}
|
||||
labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.20}
|
||||
spec:
|
||||
entrypoint: only-logistic-regression
|
||||
templates:
|
||||
- name: download-data-function
|
||||
container:
|
||||
args: []
|
||||
command: [python, download_data.py, --data, /tmp/outputs/Data/data]
|
||||
image: lightnighttw/kubeflow:download_data
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
outputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/outputs/Data/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Download
|
||||
toy data from sklearn datasets", "implementation": {"container": {"command":
|
||||
["python", "download_data.py", "--data", {"outputPath": "Data"}], "image":
|
||||
"lightnighttw/kubeflow:download_data"}}, "name": "Download Data Function",
|
||||
"outputs": [{"description": "Path where data will be stored.", "name": "Data",
|
||||
"type": "LocalPath"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
|
||||
"467750defdccfec51c3af2a7eb853f74235f5f97329006d72bf33ff6e15ed02d", "url":
|
||||
"download_data/download_data.yaml"}'}
|
||||
- name: logistic-regression-classifier
|
||||
container:
|
||||
args: []
|
||||
command: [python, logistic_regression.py, --data, /tmp/inputs/Data/data, --accuracy,
|
||||
/tmp/outputs/Accuracy/data]
|
||||
image: lightnighttw/kubeflow:logistic_regression
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/inputs/Data/data}
|
||||
outputs:
|
||||
parameters:
|
||||
- name: logistic-regression-classifier-Accuracy
|
||||
valueFrom: {path: /tmp/outputs/Accuracy/data}
|
||||
artifacts:
|
||||
- {name: logistic-regression-classifier-Accuracy, path: /tmp/outputs/Accuracy/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Trains
|
||||
a Logistic Regression Classifier", "implementation": {"container": {"command":
|
||||
["python", "logistic_regression.py", "--data", {"inputPath": "Data"}, "--accuracy",
|
||||
{"outputPath": "Accuracy"}], "image": "lightnighttw/kubeflow:logistic_regression"}},
|
||||
"inputs": [{"description": "Path where data is stored.", "name": "Data",
|
||||
"type": "LocalPath"}], "name": "Logistic Regression Classifier", "outputs":
|
||||
[{"description": "Accuracy metric", "name": "Accuracy", "type": "Float"}]}',
|
||||
pipelines.kubeflow.org/component_ref: '{"digest": "a8d1e77d07d18a75bef200aee96f35136833fc4bb535f33fd949a307beb094c2",
|
||||
"url": "logistic_regression/logistic_regression.yaml"}'}
|
||||
- name: only-logistic-regression
|
||||
dag:
|
||||
tasks:
|
||||
- {name: download-data-function, template: download-data-function}
|
||||
- name: logistic-regression-classifier
|
||||
template: logistic-regression-classifier
|
||||
dependencies: [download-data-function]
|
||||
arguments:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, from: '{{tasks.download-data-function.outputs.artifacts.download-data-function-Data}}'}
|
||||
- name: show-results
|
||||
template: show-results
|
||||
dependencies: [logistic-regression-classifier]
|
||||
arguments:
|
||||
parameters:
|
||||
- {name: logistic-regression-classifier-Accuracy, value: '{{tasks.logistic-regression-classifier.outputs.parameters.logistic-regression-classifier-Accuracy}}'}
|
||||
- name: show-results
|
||||
container:
|
||||
args: [--logistic-regression, '{{inputs.parameters.logistic-regression-classifier-Accuracy}}']
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
program_path=$(mktemp)
|
||||
printf "%s" "$0" > "$program_path"
|
||||
python3 -u "$program_path" "$@"
|
||||
- |
|
||||
def show_results(logistic_regression):
|
||||
# the results are shown.
|
||||
|
||||
print(f"Logistic regression (accuracy): {logistic_regression}")
|
||||
|
||||
import argparse
|
||||
_parser = argparse.ArgumentParser(prog='Show results', description='')
|
||||
_parser.add_argument("--logistic-regression", dest="logistic_regression", type=float, required=True, default=argparse.SUPPRESS)
|
||||
_parsed_args = vars(_parser.parse_args())
|
||||
|
||||
_outputs = show_results(**_parsed_args)
|
||||
image: python:3.7
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
parameters:
|
||||
- {name: logistic-regression-classifier-Accuracy}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container":
|
||||
{"args": ["--logistic-regression", {"inputValue": "logistic_regression"}],
|
||||
"command": ["sh", "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" >
|
||||
\"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "def show_results(logistic_regression):\n #
|
||||
the results are shown.\n\n print(f\"Logistic regression (accuracy): {logistic_regression}\")\n\nimport
|
||||
argparse\n_parser = argparse.ArgumentParser(prog=''Show results'', description='''')\n_parser.add_argument(\"--logistic-regression\",
|
||||
dest=\"logistic_regression\", type=float, required=True, default=argparse.SUPPRESS)\n_parsed_args
|
||||
= vars(_parser.parse_args())\n\n_outputs = show_results(**_parsed_args)\n"],
|
||||
"image": "python:3.7"}}, "inputs": [{"name": "logistic_regression", "type":
|
||||
"Float"}], "name": "Show results"}', pipelines.kubeflow.org/component_ref: '{}',
|
||||
pipelines.kubeflow.org/arguments.parameters: '{"logistic_regression": "{{inputs.parameters.logistic-regression-classifier-Accuracy}}"}'}
|
||||
arguments:
|
||||
parameters: []
|
||||
serviceAccountName: pipeline-runner
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Workflow
|
||||
metadata:
|
||||
generateName: three-pipeline-
|
||||
annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.20, pipelines.kubeflow.org/pipeline_compilation_time: '2023-04-28T22:03:53.972623',
|
||||
pipelines.kubeflow.org/pipeline_spec: '{"description": "Applies random forest
|
||||
for classification problem.", "name": "Three Pipeline"}'}
|
||||
labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.20}
|
||||
spec:
|
||||
entrypoint: three-pipeline
|
||||
templates:
|
||||
- name: download-data-function
|
||||
container:
|
||||
args: []
|
||||
command: [python, download_data.py, --data, /tmp/outputs/Data/data]
|
||||
image: lightnighttw/kubeflow:download_data
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
outputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/outputs/Data/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Download
|
||||
toy data from sklearn datasets", "implementation": {"container": {"command":
|
||||
["python", "download_data.py", "--data", {"outputPath": "Data"}], "image":
|
||||
"lightnighttw/kubeflow:download_data"}}, "name": "Download Data Function",
|
||||
"outputs": [{"description": "Path where data will be stored.", "name": "Data",
|
||||
"type": "LocalPath"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
|
||||
"467750defdccfec51c3af2a7eb853f74235f5f97329006d72bf33ff6e15ed02d", "url":
|
||||
"download_data/download_data.yaml"}'}
|
||||
- name: random-forest-classifier
|
||||
container:
|
||||
args: []
|
||||
command: [python, randomforest.py, --data, /tmp/inputs/Data/data, --accuracy,
|
||||
/tmp/outputs/Accuracy/data]
|
||||
image: lightnighttw/kubeflow:random_forest_v4
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/inputs/Data/data}
|
||||
outputs:
|
||||
parameters:
|
||||
- name: random-forest-classifier-Accuracy
|
||||
valueFrom: {path: /tmp/outputs/Accuracy/data}
|
||||
artifacts:
|
||||
- {name: random-forest-classifier-Accuracy, path: /tmp/outputs/Accuracy/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Trains
|
||||
a random forest classifier", "implementation": {"container": {"command":
|
||||
["python", "randomforest.py", "--data", {"inputPath": "Data"}, "--accuracy",
|
||||
{"outputPath": "Accuracy"}], "image": "lightnighttw/kubeflow:random_forest_v4"}},
|
||||
"inputs": [{"description": "Path where data is stored.", "name": "Data",
|
||||
"type": "LocalPath"}], "name": "Random Forest classifier", "outputs": [{"description":
|
||||
"Accuracy metric", "name": "Accuracy", "type": "Float"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
|
||||
"7b6a0d03d7954a2b4dc7ad9295871e1780c784799f719f7365b37f1a96a6f824", "url":
|
||||
"randomForest/random_forest.yaml"}'}
|
||||
- name: show-results
|
||||
container:
|
||||
args: [--random-forest, '{{inputs.parameters.random-forest-classifier-Accuracy}}']
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
program_path=$(mktemp)
|
||||
printf "%s" "$0" > "$program_path"
|
||||
python3 -u "$program_path" "$@"
|
||||
- |
|
||||
def show_results(random_forest):
|
||||
# the results are shown.
|
||||
|
||||
print(f"Random forest (accuracy): {random_forest}")
|
||||
|
||||
import argparse
|
||||
_parser = argparse.ArgumentParser(prog='Show results', description='')
|
||||
_parser.add_argument("--random-forest", dest="random_forest", type=float, required=True, default=argparse.SUPPRESS)
|
||||
_parsed_args = vars(_parser.parse_args())
|
||||
|
||||
_outputs = show_results(**_parsed_args)
|
||||
image: python:3.7
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
parameters:
|
||||
- {name: random-forest-classifier-Accuracy}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.20
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container":
|
||||
{"args": ["--random-forest", {"inputValue": "random_forest"}], "command":
|
||||
["sh", "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3
|
||||
-u \"$program_path\" \"$@\"\n", "def show_results(random_forest):\n #
|
||||
the results are shown.\n\n print(f\"Random forest (accuracy): {random_forest}\")\n\nimport
|
||||
argparse\n_parser = argparse.ArgumentParser(prog=''Show results'', description='''')\n_parser.add_argument(\"--random-forest\",
|
||||
dest=\"random_forest\", type=float, required=True, default=argparse.SUPPRESS)\n_parsed_args
|
||||
= vars(_parser.parse_args())\n\n_outputs = show_results(**_parsed_args)\n"],
|
||||
"image": "python:3.7"}}, "inputs": [{"name": "random_forest", "type": "Float"}],
|
||||
"name": "Show results"}', pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"random_forest":
|
||||
"{{inputs.parameters.random-forest-classifier-Accuracy}}"}'}
|
||||
- name: three-pipeline
|
||||
dag:
|
||||
tasks:
|
||||
- {name: download-data-function, template: download-data-function}
|
||||
- name: random-forest-classifier
|
||||
template: random-forest-classifier
|
||||
dependencies: [download-data-function]
|
||||
arguments:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, from: '{{tasks.download-data-function.outputs.artifacts.download-data-function-Data}}'}
|
||||
- name: show-results
|
||||
template: show-results
|
||||
dependencies: [random-forest-classifier]
|
||||
arguments:
|
||||
parameters:
|
||||
- {name: random-forest-classifier-Accuracy, value: '{{tasks.random-forest-classifier.outputs.parameters.random-forest-classifier-Accuracy}}'}
|
||||
arguments:
|
||||
parameters: []
|
||||
serviceAccountName: pipeline-runner
|
||||
|
|
@ -0,0 +1,229 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Workflow
|
||||
metadata:
|
||||
generateName: three-pipeline-
|
||||
annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.9, pipelines.kubeflow.org/pipeline_compilation_time: '2023-04-28T12:05:44.365082',
|
||||
pipelines.kubeflow.org/pipeline_spec: '{"description": "Applies Decision Tree,
|
||||
random forest and Logistic Regression for classification problem.", "name":
|
||||
"Three Pipeline"}'}
|
||||
labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.9}
|
||||
spec:
|
||||
entrypoint: three-pipeline
|
||||
templates:
|
||||
- name: decision-tree-classifier
|
||||
container:
|
||||
args: []
|
||||
command: [python, decision_tree.py, --data, /tmp/inputs/Data/data, --accuracy,
|
||||
/tmp/outputs/Accuracy/data]
|
||||
image: lightnighttw/kubeflow:decision_tree_v2
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/inputs/Data/data}
|
||||
outputs:
|
||||
parameters:
|
||||
- name: decision-tree-classifier-Accuracy
|
||||
valueFrom: {path: /tmp/outputs/Accuracy/data}
|
||||
artifacts:
|
||||
- {name: decision-tree-classifier-Accuracy, path: /tmp/outputs/Accuracy/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.9
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Trains
|
||||
a decision tree classifier", "implementation": {"container": {"command":
|
||||
["python", "decision_tree.py", "--data", {"inputPath": "Data"}, "--accuracy",
|
||||
{"outputPath": "Accuracy"}], "image": "lightnighttw/kubeflow:decision_tree_v2"}},
|
||||
"inputs": [{"description": "Path where data is stored.", "name": "Data",
|
||||
"type": "LocalPath"}], "name": "Decision Tree classifier", "outputs": [{"description":
|
||||
"Accuracy metric", "name": "Accuracy", "type": "Float"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
|
||||
"c5c232a9654213b3b222693949b71a5d561d04ed09543c20b6f8f2eab651ae0b", "url":
|
||||
"decision_tree/decision_tree.yaml"}'}
|
||||
- name: download-data-function
|
||||
container:
|
||||
args: []
|
||||
command: [python, download_data.py, --data, /tmp/outputs/Data/data]
|
||||
image: lightnighttw/kubeflow:download_data
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
outputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/outputs/Data/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.9
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Download
|
||||
toy data from sklearn datasets", "implementation": {"container": {"command":
|
||||
["python", "download_data.py", "--data", {"outputPath": "Data"}], "image":
|
||||
"lightnighttw/kubeflow:download_data"}}, "name": "Download Data Function",
|
||||
"outputs": [{"description": "Path where data will be stored.", "name": "Data",
|
||||
"type": "LocalPath"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
|
||||
"467750defdccfec51c3af2a7eb853f74235f5f97329006d72bf33ff6e15ed02d", "url":
|
||||
"download_data/download_data.yaml"}'}
|
||||
- name: logistic-regression-classifier
|
||||
container:
|
||||
args: []
|
||||
command: [python, logistic_regression.py, --data, /tmp/inputs/Data/data, --accuracy,
|
||||
/tmp/outputs/Accuracy/data]
|
||||
image: lightnighttw/kubeflow:logistic_regression
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/inputs/Data/data}
|
||||
outputs:
|
||||
parameters:
|
||||
- name: logistic-regression-classifier-Accuracy
|
||||
valueFrom: {path: /tmp/outputs/Accuracy/data}
|
||||
artifacts:
|
||||
- {name: logistic-regression-classifier-Accuracy, path: /tmp/outputs/Accuracy/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.9
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Trains
|
||||
a Logistic Regression Classifier", "implementation": {"container": {"command":
|
||||
["python", "logistic_regression.py", "--data", {"inputPath": "Data"}, "--accuracy",
|
||||
{"outputPath": "Accuracy"}], "image": "lightnighttw/kubeflow:logistic_regression"}},
|
||||
"inputs": [{"description": "Path where data is stored.", "name": "Data",
|
||||
"type": "LocalPath"}], "name": "Logistic Regression Classifier", "outputs":
|
||||
[{"description": "Accuracy metric", "name": "Accuracy", "type": "Float"}]}',
|
||||
pipelines.kubeflow.org/component_ref: '{"digest": "a8d1e77d07d18a75bef200aee96f35136833fc4bb535f33fd949a307beb094c2",
|
||||
"url": "logistic_regression/logistic_regression.yaml"}'}
|
||||
- name: random-forest-classifier
|
||||
container:
|
||||
args: []
|
||||
command: [python, randomforest.py, --data, /tmp/inputs/Data/data, --accuracy,
|
||||
/tmp/outputs/Accuracy/data]
|
||||
image: lightnighttw/kubeflow:random_forest_v4
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, path: /tmp/inputs/Data/data}
|
||||
outputs:
|
||||
parameters:
|
||||
- name: random-forest-classifier-Accuracy
|
||||
valueFrom: {path: /tmp/outputs/Accuracy/data}
|
||||
artifacts:
|
||||
- {name: random-forest-classifier-Accuracy, path: /tmp/outputs/Accuracy/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.9
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Train
|
||||
a random forest classifier", "implementation": {"container": {"command":
|
||||
["python", "randomforest.py", "--data", {"inputPath": "Data"}, "--accuracy",
|
||||
{"outputPath": "Accuracy"}], "image": "lightnighttw/kubeflow:random_forest_v4"}},
|
||||
"inputs": [{"description": "Path where data is stored.", "name": "Data",
|
||||
"type": "LocalPath"}], "name": "Random Forest classifier", "outputs": [{"description":
|
||||
"Accuracy metric", "name": "Accuracy", "type": "Float"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
|
||||
"b49b12da3371976eddf41d662685bb49d71b419d516de65efdd90938d2c706bc", "url":
|
||||
"randomForest/random_forest.yaml"}'}
|
||||
- name: show-results
|
||||
container:
|
||||
args: [--decision-tree, '{{inputs.parameters.decision-tree-classifier-Accuracy}}',
|
||||
--logistic-regression, '{{inputs.parameters.logistic-regression-classifier-Accuracy}}',
|
||||
--random-forest, '{{inputs.parameters.random-forest-classifier-Accuracy}}']
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
program_path=$(mktemp)
|
||||
printf "%s" "$0" > "$program_path"
|
||||
python3 -u "$program_path" "$@"
|
||||
- |
|
||||
def show_results(decision_tree, logistic_regression, random_forest):
|
||||
# Given the outputs from decision_tree and logistic regression components
|
||||
# the results are shown.
|
||||
|
||||
print(f"Decision tree (accuracy): {decision_tree}")
|
||||
print(f"Logistic regression (accuracy): {logistic_regression}")
|
||||
print(f"Random forest (accuracy): {random_forest}")
|
||||
|
||||
import argparse
|
||||
_parser = argparse.ArgumentParser(prog='Show results', description='')
|
||||
_parser.add_argument("--decision-tree", dest="decision_tree", type=float, required=True, default=argparse.SUPPRESS)
|
||||
_parser.add_argument("--logistic-regression", dest="logistic_regression", type=float, required=True, default=argparse.SUPPRESS)
|
||||
_parser.add_argument("--random-forest", dest="random_forest", type=float, required=True, default=argparse.SUPPRESS)
|
||||
_parsed_args = vars(_parser.parse_args())
|
||||
|
||||
_outputs = show_results(**_parsed_args)
|
||||
image: python:3.7
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
inputs:
|
||||
parameters:
|
||||
- {name: decision-tree-classifier-Accuracy}
|
||||
- {name: logistic-regression-classifier-Accuracy}
|
||||
- {name: random-forest-classifier-Accuracy}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.8.9
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container":
|
||||
{"args": ["--decision-tree", {"inputValue": "decision_tree"}, "--logistic-regression",
|
||||
{"inputValue": "logistic_regression"}, "--random-forest", {"inputValue":
|
||||
"random_forest"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf
|
||||
\"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n",
|
||||
"def show_results(decision_tree, logistic_regression, random_forest):\n #
|
||||
Given the outputs from decision_tree and logistic regression components\n #
|
||||
the results are shown.\n\n print(f\"Decision tree (accuracy): {decision_tree}\")\n print(f\"Logistic
|
||||
regression (accuracy): {logistic_regression}\")\n print(f\"Random forest
|
||||
(accuracy): {random_forest}\")\n\nimport argparse\n_parser = argparse.ArgumentParser(prog=''Show
|
||||
results'', description='''')\n_parser.add_argument(\"--decision-tree\",
|
||||
dest=\"decision_tree\", type=float, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--logistic-regression\",
|
||||
dest=\"logistic_regression\", type=float, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--random-forest\",
|
||||
dest=\"random_forest\", type=float, required=True, default=argparse.SUPPRESS)\n_parsed_args
|
||||
= vars(_parser.parse_args())\n\n_outputs = show_results(**_parsed_args)\n"],
|
||||
"image": "python:3.7"}}, "inputs": [{"name": "decision_tree", "type": "Float"},
|
||||
{"name": "logistic_regression", "type": "Float"}, {"name": "random_forest",
|
||||
"type": "Float"}], "name": "Show results"}', pipelines.kubeflow.org/component_ref: '{}',
|
||||
pipelines.kubeflow.org/arguments.parameters: '{"decision_tree": "{{inputs.parameters.decision-tree-classifier-Accuracy}}",
|
||||
"logistic_regression": "{{inputs.parameters.logistic-regression-classifier-Accuracy}}",
|
||||
"random_forest": "{{inputs.parameters.random-forest-classifier-Accuracy}}"}'}
|
||||
- name: three-pipeline
|
||||
dag:
|
||||
tasks:
|
||||
- name: decision-tree-classifier
|
||||
template: decision-tree-classifier
|
||||
dependencies: [download-data-function]
|
||||
arguments:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, from: '{{tasks.download-data-function.outputs.artifacts.download-data-function-Data}}'}
|
||||
- {name: download-data-function, template: download-data-function}
|
||||
- name: logistic-regression-classifier
|
||||
template: logistic-regression-classifier
|
||||
dependencies: [download-data-function]
|
||||
arguments:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, from: '{{tasks.download-data-function.outputs.artifacts.download-data-function-Data}}'}
|
||||
- name: random-forest-classifier
|
||||
template: random-forest-classifier
|
||||
dependencies: [download-data-function]
|
||||
arguments:
|
||||
artifacts:
|
||||
- {name: download-data-function-Data, from: '{{tasks.download-data-function.outputs.artifacts.download-data-function-Data}}'}
|
||||
- name: show-results
|
||||
template: show-results
|
||||
dependencies: [decision-tree-classifier, logistic-regression-classifier, random-forest-classifier]
|
||||
arguments:
|
||||
parameters:
|
||||
- {name: decision-tree-classifier-Accuracy, value: '{{tasks.decision-tree-classifier.outputs.parameters.decision-tree-classifier-Accuracy}}'}
|
||||
- {name: logistic-regression-classifier-Accuracy, value: '{{tasks.logistic-regression-classifier.outputs.parameters.logistic-regression-classifier-Accuracy}}'}
|
||||
- {name: random-forest-classifier-Accuracy, value: '{{tasks.random-forest-classifier.outputs.parameters.random-forest-classifier-Accuracy}}'}
|
||||
arguments:
|
||||
parameters: []
|
||||
serviceAccountName: pipeline-runner
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
setuptools
|
||||
wheel
|
||||
PyYAML==5.3.1
|
||||
minio
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Workflow
|
||||
metadata:
|
||||
generateName: kfserving-pipeline-
|
||||
annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.7.0, pipelines.kubeflow.org/pipeline_compilation_time: '2023-11-14T09:23:34.926467',
|
||||
pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline for KFServing
|
||||
with PVC.", "inputs": [{"default": "apply", "name": "action", "optional": true},
|
||||
{"default": "kubeflow02", "name": "namespace", "optional": true}, {"default":
|
||||
"undefined", "name": "pvc_name", "optional": true}, {"default": "model01", "name":
|
||||
"model_name", "optional": true}], "name": "KFServing pipeline"}'}
|
||||
labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.7.0}
|
||||
spec:
|
||||
entrypoint: kfserving-pipeline
|
||||
templates:
|
||||
- name: kfserving-pipeline
|
||||
inputs:
|
||||
parameters:
|
||||
- {name: action}
|
||||
- {name: model_name}
|
||||
- {name: namespace}
|
||||
- {name: pvc_name}
|
||||
dag:
|
||||
tasks:
|
||||
- name: serve-a-model-with-kserve
|
||||
template: serve-a-model-with-kserve
|
||||
arguments:
|
||||
parameters:
|
||||
- {name: action, value: '{{inputs.parameters.action}}'}
|
||||
- {name: model_name, value: '{{inputs.parameters.model_name}}'}
|
||||
- {name: namespace, value: '{{inputs.parameters.namespace}}'}
|
||||
- {name: pvc_name, value: '{{inputs.parameters.pvc_name}}'}
|
||||
- name: serve-a-model-with-kserve
|
||||
container:
|
||||
args:
|
||||
- -u
|
||||
- kservedeployer.py
|
||||
- --action
|
||||
- '{{inputs.parameters.action}}'
|
||||
- --model-name
|
||||
- ''
|
||||
- --model-uri
|
||||
- ''
|
||||
- --canary-traffic-percent
|
||||
- '100'
|
||||
- --namespace
|
||||
- ''
|
||||
- --framework
|
||||
- ''
|
||||
- --runtime-version
|
||||
- latest
|
||||
- --resource-requests
|
||||
- '{"cpu": "0.5", "memory": "512Mi"}'
|
||||
- --resource-limits
|
||||
- '{"cpu": "1", "memory": "1Gi"}'
|
||||
- --custom-model-spec
|
||||
- '{}'
|
||||
- --autoscaling-target
|
||||
- '0'
|
||||
- --service-account
|
||||
- ''
|
||||
- --enable-istio-sidecar
|
||||
- "True"
|
||||
- --output-path
|
||||
- /tmp/outputs/InferenceService_Status/data
|
||||
- --inferenceservice-yaml
|
||||
- |2
|
||||
|
||||
apiVersion: "serving.kserve.io/v1beta1"
|
||||
kind: "InferenceService"
|
||||
metadata:
|
||||
name: {{inputs.parameters.model_name}}
|
||||
namespace: {{inputs.parameters.namespace}}
|
||||
spec:
|
||||
predictor:
|
||||
sklearn:
|
||||
storageUri: pvc://{{inputs.parameters.pvc_name}}/{{inputs.parameters.model_name}}/
|
||||
resources:
|
||||
limits:
|
||||
cpu: "100m"
|
||||
requests:
|
||||
cpu: "100m"
|
||||
- --watch-timeout
|
||||
- '300'
|
||||
- --min-replicas
|
||||
- '-1'
|
||||
- --max-replicas
|
||||
- '-1'
|
||||
- --request-timeout
|
||||
- '60'
|
||||
- --enable-isvc-status
|
||||
- "True"
|
||||
command: [python]
|
||||
image: quay.io/aipipeline/kserve-component:v0.11.1
|
||||
resources:
|
||||
limits: {cpu: 500m}
|
||||
requests: {cpu: 500m}
|
||||
inputs:
|
||||
parameters:
|
||||
- {name: action}
|
||||
- {name: model_name}
|
||||
- {name: namespace}
|
||||
- {name: pvc_name}
|
||||
outputs:
|
||||
artifacts:
|
||||
- {name: serve-a-model-with-kserve-InferenceService-Status, path: /tmp/outputs/InferenceService_Status/data}
|
||||
metadata:
|
||||
labels:
|
||||
pipelines.kubeflow.org/kfp_sdk_version: 1.7.0
|
||||
pipelines.kubeflow.org/pipeline-sdk-type: kfp
|
||||
pipelines.kubeflow.org/enable_caching: "true"
|
||||
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Serve
|
||||
Models using KServe", "implementation": {"container": {"args": ["-u", "kservedeployer.py",
|
||||
"--action", {"inputValue": "Action"}, "--model-name", {"inputValue": "Model
|
||||
Name"}, "--model-uri", {"inputValue": "Model URI"}, "--canary-traffic-percent",
|
||||
{"inputValue": "Canary Traffic Percent"}, "--namespace", {"inputValue":
|
||||
"Namespace"}, "--framework", {"inputValue": "Framework"}, "--runtime-version",
|
||||
{"inputValue": "Runtime Version"}, "--resource-requests", {"inputValue":
|
||||
"Resource Requests"}, "--resource-limits", {"inputValue": "Resource Limits"},
|
||||
"--custom-model-spec", {"inputValue": "Custom Model Spec"}, "--autoscaling-target",
|
||||
{"inputValue": "Autoscaling Target"}, "--service-account", {"inputValue":
|
||||
"Service Account"}, "--enable-istio-sidecar", {"inputValue": "Enable Istio
|
||||
Sidecar"}, "--output-path", {"outputPath": "InferenceService Status"}, "--inferenceservice-yaml",
|
||||
{"inputValue": "InferenceService YAML"}, "--watch-timeout", {"inputValue":
|
||||
"Watch Timeout"}, "--min-replicas", {"inputValue": "Min Replicas"}, "--max-replicas",
|
||||
{"inputValue": "Max Replicas"}, "--request-timeout", {"inputValue": "Request
|
||||
Timeout"}, "--enable-isvc-status", {"inputValue": "Enable ISVC Status"}],
|
||||
"command": ["python"], "image": "quay.io/aipipeline/kserve-component:v0.11.1"}},
|
||||
"inputs": [{"default": "create", "description": "Action to execute on KServe",
|
||||
"name": "Action", "type": "String"}, {"default": "", "description": "Name
|
||||
to give to the deployed model", "name": "Model Name", "type": "String"},
|
||||
{"default": "", "description": "Path of the S3 or GCS compatible directory
|
||||
containing the model.", "name": "Model URI", "type": "String"}, {"default":
|
||||
"100", "description": "The traffic split percentage between the candidate
|
||||
model and the last ready model", "name": "Canary Traffic Percent", "type":
|
||||
"String"}, {"default": "", "description": "Kubernetes namespace where the
|
||||
KServe service is deployed.", "name": "Namespace", "type": "String"}, {"default":
|
||||
"", "description": "Machine Learning Framework for Model Serving.", "name":
|
||||
"Framework", "type": "String"}, {"default": "latest", "description": "Runtime
|
||||
Version of Machine Learning Framework", "name": "Runtime Version", "type":
|
||||
"String"}, {"default": "{\"cpu\": \"0.5\", \"memory\": \"512Mi\"}", "description":
|
||||
"CPU and Memory requests for Model Serving", "name": "Resource Requests",
|
||||
"type": "String"}, {"default": "{\"cpu\": \"1\", \"memory\": \"1Gi\"}",
|
||||
"description": "CPU and Memory limits for Model Serving", "name": "Resource
|
||||
Limits", "type": "String"}, {"default": "{}", "description": "Custom model
|
||||
runtime container spec in JSON", "name": "Custom Model Spec", "type": "String"},
|
||||
{"default": "0", "description": "Autoscaling Target Number", "name": "Autoscaling
|
||||
Target", "type": "String"}, {"default": "", "description": "ServiceAccount
|
||||
to use to run the InferenceService pod", "name": "Service Account", "type":
|
||||
"String"}, {"default": "True", "description": "Whether to enable istio sidecar
|
||||
injection", "name": "Enable Istio Sidecar", "type": "Bool"}, {"default":
|
||||
"{}", "description": "Raw InferenceService serialized YAML for deployment",
|
||||
"name": "InferenceService YAML", "type": "String"}, {"default": "300", "description":
|
||||
"Timeout seconds for watching until InferenceService becomes ready.", "name":
|
||||
"Watch Timeout", "type": "String"}, {"default": "-1", "description": "Minimum
|
||||
number of InferenceService replicas", "name": "Min Replicas", "type": "String"},
|
||||
{"default": "-1", "description": "Maximum number of InferenceService replicas",
|
||||
"name": "Max Replicas", "type": "String"}, {"default": "60", "description":
|
||||
"Specifies the number of seconds to wait before timing out a request to
|
||||
the component.", "name": "Request Timeout", "type": "String"}, {"default":
|
||||
"True", "description": "Specifies whether to store the inference service
|
||||
status as the output parameter", "name": "Enable ISVC Status", "type": "Bool"}],
|
||||
"name": "Serve a model with KServe", "outputs": [{"description": "Status
|
||||
JSON output of InferenceService", "name": "InferenceService Status", "type":
|
||||
"String"}]}', pipelines.kubeflow.org/component_ref: '{"digest": "b0379af21c170410b8b1a9606cb6cad63f99b0af53c2a5c1f0af397b53c81cd7",
|
||||
"url": "https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kserve/component.yaml"}',
|
||||
pipelines.kubeflow.org/arguments.parameters: '{"Action": "{{inputs.parameters.action}}",
|
||||
"Autoscaling Target": "0", "Canary Traffic Percent": "100", "Custom Model
|
||||
Spec": "{}", "Enable ISVC Status": "True", "Enable Istio Sidecar": "True",
|
||||
"Framework": "", "InferenceService YAML": "\napiVersion: \"serving.kserve.io/v1beta1\"\nkind:
|
||||
\"InferenceService\"\nmetadata:\n name: {{inputs.parameters.model_name}}\n namespace:
|
||||
{{inputs.parameters.namespace}}\nspec:\n predictor:\n sklearn:\n storageUri:
|
||||
pvc://{{inputs.parameters.pvc_name}}/{{inputs.parameters.model_name}}/\n resources:\n limits:\n cpu:
|
||||
\"100m\"\n requests:\n cpu: \"100m\"\n", "Max Replicas":
|
||||
"-1", "Min Replicas": "-1", "Model Name": "", "Model URI": "", "Namespace":
|
||||
"", "Request Timeout": "60", "Resource Limits": "{\"cpu\": \"1\", \"memory\":
|
||||
\"1Gi\"}", "Resource Requests": "{\"cpu\": \"0.5\", \"memory\": \"512Mi\"}",
|
||||
"Runtime Version": "latest", "Service Account": "", "Watch Timeout": "300"}'}
|
||||
arguments:
|
||||
parameters:
|
||||
- {name: action, value: apply}
|
||||
- {name: namespace, value: kubeflow02}
|
||||
- {name: pvc_name, value: undefined}
|
||||
- {name: model_name, value: model01}
|
||||
serviceAccountName: pipeline-runner
|
||||
|
|
@ -0,0 +1,498 @@
|
|||
/**
|
||||
* This is the default settings file provided by Node-RED.
|
||||
*
|
||||
* It can contain any valid JavaScript code that will get run when Node-RED
|
||||
* is started.
|
||||
*
|
||||
* Lines that start with // are commented out.
|
||||
* Each entry should be separated from the entries above and below by a comma ','
|
||||
*
|
||||
* For more information about individual settings, refer to the documentation:
|
||||
* https://nodered.org/docs/user-guide/runtime/configuration
|
||||
*
|
||||
* The settings are split into the following sections:
|
||||
* - Flow File and User Directory Settings
|
||||
* - Security
|
||||
* - Server Settings
|
||||
* - Runtime Settings
|
||||
* - Editor Settings
|
||||
* - Node Settings
|
||||
*
|
||||
**/
|
||||
|
||||
module.exports = {
|
||||
|
||||
/*******************************************************************************
|
||||
* Flow File and User Directory Settings
|
||||
* - flowFile
|
||||
* - credentialSecret
|
||||
* - flowFilePretty
|
||||
* - userDir
|
||||
* - nodesDir
|
||||
******************************************************************************/
|
||||
|
||||
/** The file containing the flows. If not set, defaults to flows_<hostname>.json **/
|
||||
flowFile: 'flows.json',
|
||||
|
||||
/** By default, credentials are encrypted in storage using a generated key. To
|
||||
* specify your own secret, set the following property.
|
||||
* If you want to disable encryption of credentials, set this property to false.
|
||||
* Note: once you set this property, do not change it - doing so will prevent
|
||||
* node-red from being able to decrypt your existing credentials and they will be
|
||||
* lost.
|
||||
*/
|
||||
//credentialSecret: "a-secret-key",
|
||||
credentialSecret: process.env.NODE_RED_CREDENTIAL_SECRET,
|
||||
|
||||
/** By default, the flow JSON will be formatted over multiple lines making
|
||||
* it easier to compare changes when using version control.
|
||||
* To disable pretty-printing of the JSON set the following property to false.
|
||||
*/
|
||||
flowFilePretty: true,
|
||||
|
||||
/** By default, all user data is stored in a directory called `.node-red` under
|
||||
* the user's home directory. To use a different location, the following
|
||||
* property can be used
|
||||
*/
|
||||
//userDir: '/home/nol/.node-red/',
|
||||
|
||||
/** Node-RED scans the `nodes` directory in the userDir to find local node files.
|
||||
* The following property can be used to specify an additional directory to scan.
|
||||
*/
|
||||
//nodesDir: '/home/nol/.node-red/nodes',
|
||||
|
||||
/*******************************************************************************
|
||||
* Security
|
||||
* - adminAuth
|
||||
* - https
|
||||
* - httpsRefreshInterval
|
||||
* - requireHttps
|
||||
* - httpNodeAuth
|
||||
* - httpStaticAuth
|
||||
******************************************************************************/
|
||||
|
||||
/** To password protect the Node-RED editor and admin API, the following
|
||||
* property can be used. See http://nodered.org/docs/security.html for details.
|
||||
*/
|
||||
//adminAuth: {
|
||||
// type: "credentials",
|
||||
// users: [{
|
||||
// username: "admin",
|
||||
// password: "$2a$08$zZWtXTja0fB1pzD4sHCMyOCMYz2Z6dNbM6tl8sJogENOMcxWV9DN.",
|
||||
// permissions: "*"
|
||||
// }]
|
||||
//},
|
||||
|
||||
/** The following property can be used to enable HTTPS
|
||||
* This property can be either an object, containing both a (private) key
|
||||
* and a (public) certificate, or a function that returns such an object.
|
||||
* See http://nodejs.org/api/https.html#https_https_createserver_options_requestlistener
|
||||
* for details of its contents.
|
||||
*/
|
||||
|
||||
/** Option 1: static object */
|
||||
//https: {
|
||||
// key: require("fs").readFileSync('privkey.pem'),
|
||||
// cert: require("fs").readFileSync('cert.pem')
|
||||
//},
|
||||
|
||||
/** Option 2: function that returns the HTTP configuration object */
|
||||
// https: function() {
|
||||
// // This function should return the options object, or a Promise
|
||||
// // that resolves to the options object
|
||||
// return {
|
||||
// key: require("fs").readFileSync('privkey.pem'),
|
||||
// cert: require("fs").readFileSync('cert.pem')
|
||||
// }
|
||||
// },
|
||||
|
||||
/** If the `https` setting is a function, the following setting can be used
|
||||
* to set how often, in hours, the function will be called. That can be used
|
||||
* to refresh any certificates.
|
||||
*/
|
||||
//httpsRefreshInterval : 12,
|
||||
|
||||
/** The following property can be used to cause insecure HTTP connections to
|
||||
* be redirected to HTTPS.
|
||||
*/
|
||||
//requireHttps: true,
|
||||
|
||||
/** To password protect the node-defined HTTP endpoints (httpNodeRoot),
|
||||
* including node-red-dashboard, or the static content (httpStatic), the
|
||||
* following properties can be used.
|
||||
* The `pass` field is a bcrypt hash of the password.
|
||||
* See http://nodered.org/docs/security.html#generating-the-password-hash
|
||||
*/
|
||||
//httpNodeAuth: {user:"user",pass:"$2a$08$zZWtXTja0fB1pzD4sHCMyOCMYz2Z6dNbM6tl8sJogENOMcxWV9DN."},
|
||||
//httpStaticAuth: {user:"user",pass:"$2a$08$zZWtXTja0fB1pzD4sHCMyOCMYz2Z6dNbM6tl8sJogENOMcxWV9DN."},
|
||||
|
||||
/*******************************************************************************
|
||||
* Server Settings
|
||||
* - uiPort
|
||||
* - uiHost
|
||||
* - apiMaxLength
|
||||
* - httpServerOptions
|
||||
* - httpAdminRoot
|
||||
* - httpAdminMiddleware
|
||||
* - httpNodeRoot
|
||||
* - httpNodeCors
|
||||
* - httpNodeMiddleware
|
||||
* - httpStatic
|
||||
******************************************************************************/
|
||||
|
||||
/** the tcp port that the Node-RED web server is listening on */
|
||||
uiPort: process.env.PORT || 1880,
|
||||
|
||||
/** By default, the Node-RED UI accepts connections on all IPv4 interfaces.
|
||||
* To listen on all IPv6 addresses, set uiHost to "::",
|
||||
* The following property can be used to listen on a specific interface. For
|
||||
* example, the following would only allow connections from the local machine.
|
||||
*/
|
||||
//uiHost: "127.0.0.1",
|
||||
|
||||
/** The maximum size of HTTP request that will be accepted by the runtime api.
|
||||
* Default: 5mb
|
||||
*/
|
||||
//apiMaxLength: '5mb',
|
||||
|
||||
/** The following property can be used to pass custom options to the Express.js
|
||||
* server used by Node-RED. For a full list of available options, refer
|
||||
* to http://expressjs.com/en/api.html#app.settings.table
|
||||
*/
|
||||
//httpServerOptions: { },
|
||||
|
||||
/** By default, the Node-RED UI is available at http://localhost:1880/
|
||||
* The following property can be used to specify a different root path.
|
||||
* If set to false, this is disabled.
|
||||
*/
|
||||
//httpAdminRoot: '/admin',
|
||||
|
||||
/** The following property can be used to add a custom middleware function
|
||||
* in front of all admin http routes. For example, to set custom http
|
||||
* headers. It can be a single function or an array of middleware functions.
|
||||
*/
|
||||
// httpAdminMiddleware: function(req,res,next) {
|
||||
// // Set the X-Frame-Options header to limit where the editor
|
||||
// // can be embedded
|
||||
// //res.set('X-Frame-Options', 'sameorigin');
|
||||
// next();
|
||||
// },
|
||||
|
||||
|
||||
/** Some nodes, such as HTTP In, can be used to listen for incoming http requests.
|
||||
* By default, these are served relative to '/'. The following property
|
||||
* can be used to specifiy a different root path. If set to false, this is
|
||||
* disabled.
|
||||
*/
|
||||
//httpNodeRoot: '/red-nodes',
|
||||
|
||||
/** The following property can be used to configure cross-origin resource sharing
|
||||
* in the HTTP nodes.
|
||||
* See https://github.com/troygoode/node-cors#configuration-options for
|
||||
* details on its contents. The following is a basic permissive set of options:
|
||||
*/
|
||||
//httpNodeCors: {
|
||||
// origin: "*",
|
||||
// methods: "GET,PUT,POST,DELETE"
|
||||
//},
|
||||
|
||||
/** If you need to set an http proxy please set an environment variable
|
||||
* called http_proxy (or HTTP_PROXY) outside of Node-RED in the operating system.
|
||||
* For example - http_proxy=http://myproxy.com:8080
|
||||
* (Setting it here will have no effect)
|
||||
* You may also specify no_proxy (or NO_PROXY) to supply a comma separated
|
||||
* list of domains to not proxy, eg - no_proxy=.acme.co,.acme.co.uk
|
||||
*/
|
||||
|
||||
/** The following property can be used to add a custom middleware function
|
||||
* in front of all http in nodes. This allows custom authentication to be
|
||||
* applied to all http in nodes, or any other sort of common request processing.
|
||||
* It can be a single function or an array of middleware functions.
|
||||
*/
|
||||
//httpNodeMiddleware: function(req,res,next) {
|
||||
// // Handle/reject the request, or pass it on to the http in node by calling next();
|
||||
// // Optionally skip our rawBodyParser by setting this to true;
|
||||
// //req.skipRawBodyParser = true;
|
||||
// next();
|
||||
//},
|
||||
|
||||
/** When httpAdminRoot is used to move the UI to a different root path, the
|
||||
* following property can be used to identify a directory of static content
|
||||
* that should be served at http://localhost:1880/.
|
||||
*/
|
||||
//httpStatic: '/home/nol/node-red-static/',
|
||||
|
||||
/*******************************************************************************
|
||||
* Runtime Settings
|
||||
* - lang
|
||||
* - logging
|
||||
* - contextStorage
|
||||
* - exportGlobalContextKeys
|
||||
* - externalModules
|
||||
******************************************************************************/
|
||||
|
||||
/** Uncomment the following to run node-red in your preferred language.
|
||||
* Available languages include: en-US (default), ja, de, zh-CN, zh-TW, ru, ko
|
||||
* Some languages are more complete than others.
|
||||
*/
|
||||
// lang: "de",
|
||||
|
||||
/** Configure the logging output */
|
||||
logging: {
|
||||
/** Only console logging is currently supported */
|
||||
console: {
|
||||
/** Level of logging to be recorded. Options are:
|
||||
* fatal - only those errors which make the application unusable should be recorded
|
||||
* error - record errors which are deemed fatal for a particular request + fatal errors
|
||||
* warn - record problems which are non fatal + errors + fatal errors
|
||||
* info - record information about the general running of the application + warn + error + fatal errors
|
||||
* debug - record information which is more verbose than info + info + warn + error + fatal errors
|
||||
* trace - record very detailed logging + debug + info + warn + error + fatal errors
|
||||
* off - turn off all logging (doesn't affect metrics or audit)
|
||||
*/
|
||||
level: "info",
|
||||
/** Whether or not to include metric events in the log output */
|
||||
metrics: false,
|
||||
/** Whether or not to include audit events in the log output */
|
||||
audit: false
|
||||
}
|
||||
},
|
||||
|
||||
/** Context Storage
|
||||
* The following property can be used to enable context storage. The configuration
|
||||
* provided here will enable file-based context that flushes to disk every 30 seconds.
|
||||
* Refer to the documentation for further options: https://nodered.org/docs/api/context/
|
||||
*/
|
||||
//contextStorage: {
|
||||
// default: {
|
||||
// module:"localfilesystem"
|
||||
// },
|
||||
//},
|
||||
|
||||
/** `global.keys()` returns a list of all properties set in global context.
|
||||
* This allows them to be displayed in the Context Sidebar within the editor.
|
||||
* In some circumstances it is not desirable to expose them to the editor. The
|
||||
* following property can be used to hide any property set in `functionGlobalContext`
|
||||
* from being list by `global.keys()`.
|
||||
* By default, the property is set to false to avoid accidental exposure of
|
||||
* their values. Setting this to true will cause the keys to be listed.
|
||||
*/
|
||||
exportGlobalContextKeys: false,
|
||||
|
||||
/** Configure how the runtime will handle external npm modules.
|
||||
* This covers:
|
||||
* - whether the editor will allow new node modules to be installed
|
||||
* - whether nodes, such as the Function node are allowed to have their
|
||||
* own dynamically configured dependencies.
|
||||
* The allow/denyList options can be used to limit what modules the runtime
|
||||
* will install/load. It can use '*' as a wildcard that matches anything.
|
||||
*/
|
||||
externalModules: {
|
||||
// autoInstall: false, /** Whether the runtime will attempt to automatically install missing modules */
|
||||
// autoInstallRetry: 30, /** Interval, in seconds, between reinstall attempts */
|
||||
// palette: { /** Configuration for the Palette Manager */
|
||||
// allowInstall: true, /** Enable the Palette Manager in the editor */
|
||||
// allowUpdate: true, /** Allow modules to be updated in the Palette Manager */
|
||||
// allowUpload: true, /** Allow module tgz files to be uploaded and installed */
|
||||
// allowList: ['*'],
|
||||
// denyList: [],
|
||||
// allowUpdateList: ['*'],
|
||||
// denyUpdateList: []
|
||||
// },
|
||||
// modules: { /** Configuration for node-specified modules */
|
||||
// allowInstall: true,
|
||||
// allowList: [],
|
||||
// denyList: []
|
||||
// }
|
||||
},
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
* Editor Settings
|
||||
* - disableEditor
|
||||
* - editorTheme
|
||||
******************************************************************************/
|
||||
|
||||
/** The following property can be used to disable the editor. The admin API
|
||||
* is not affected by this option. To disable both the editor and the admin
|
||||
* API, use either the httpRoot or httpAdminRoot properties
|
||||
*/
|
||||
//disableEditor: false,
|
||||
|
||||
/** Customising the editor
|
||||
* See https://nodered.org/docs/user-guide/runtime/configuration#editor-themes
|
||||
* for all available options.
|
||||
*/
|
||||
editorTheme: {
|
||||
/** The following property can be used to set a custom theme for the editor.
|
||||
* See https://github.com/node-red-contrib-themes/theme-collection for
|
||||
* a collection of themes to chose from.
|
||||
*/
|
||||
//theme: "",
|
||||
|
||||
/** To disable the 'Welcome to Node-RED' tour that is displayed the first
|
||||
* time you access the editor for each release of Node-RED, set this to false
|
||||
*/
|
||||
//tours: false,
|
||||
|
||||
palette: {
|
||||
/** The following property can be used to order the categories in the editor
|
||||
* palette. If a node's category is not in the list, the category will get
|
||||
* added to the end of the palette.
|
||||
* If not set, the following default order is used:
|
||||
*/
|
||||
//categories: ['subflows', 'common', 'function', 'network', 'sequence', 'parser', 'storage'],
|
||||
},
|
||||
|
||||
projects: {
|
||||
/** To enable the Projects feature, set this value to true */
|
||||
enabled: false,
|
||||
workflow: {
|
||||
/** Set the default projects workflow mode.
|
||||
* - manual - you must manually commit changes
|
||||
* - auto - changes are automatically committed
|
||||
* This can be overridden per-user from the 'Git config'
|
||||
* section of 'User Settings' within the editor
|
||||
*/
|
||||
mode: "manual"
|
||||
}
|
||||
},
|
||||
|
||||
codeEditor: {
|
||||
/** Select the text editor component used by the editor.
|
||||
* Defaults to "ace", but can be set to "ace" or "monaco"
|
||||
*/
|
||||
lib: "ace",
|
||||
options: {
|
||||
/** The follow options only apply if the editor is set to "monaco"
|
||||
*
|
||||
* theme - must match the file name of a theme in
|
||||
* packages/node_modules/@node-red/editor-client/src/vendor/monaco/dist/theme
|
||||
* e.g. "tomorrow-night", "upstream-sunburst", "github", "my-theme"
|
||||
*/
|
||||
theme: "vs",
|
||||
/** other overrides can be set e.g. fontSize, fontFamily, fontLigatures etc.
|
||||
* for the full list, see https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.istandaloneeditorconstructionoptions.html
|
||||
*/
|
||||
//fontSize: 14,
|
||||
//fontFamily: "Cascadia Code, Fira Code, Consolas, 'Courier New', monospace",
|
||||
//fontLigatures: true,
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
/*******************************************************************************
|
||||
* Node Settings
|
||||
* - fileWorkingDirectory
|
||||
* - functionGlobalContext
|
||||
* - functionExternalModules
|
||||
* - nodeMessageBufferMaxLength
|
||||
* - ui (for use with Node-RED Dashboard)
|
||||
* - debugUseColors
|
||||
* - debugMaxLength
|
||||
* - execMaxBufferSize
|
||||
* - httpRequestTimeout
|
||||
* - mqttReconnectTime
|
||||
* - serialReconnectTime
|
||||
* - socketReconnectTime
|
||||
* - socketTimeout
|
||||
* - tcpMsgQueueSize
|
||||
* - inboundWebSocketTimeout
|
||||
* - tlsConfigDisableLocalFiles
|
||||
* - webSocketNodeVerifyClient
|
||||
******************************************************************************/
|
||||
|
||||
/** The working directory to handle relative file paths from within the File nodes
|
||||
* defaults to the working directory of the Node-RED process.
|
||||
*/
|
||||
//fileWorkingDirectory: "",
|
||||
|
||||
/** Allow the Function node to load additional npm modules directly */
|
||||
functionExternalModules: true,
|
||||
|
||||
/** The following property can be used to set predefined values in Global Context.
|
||||
* This allows extra node modules to be made available with in Function node.
|
||||
* For example, the following:
|
||||
* functionGlobalContext: { os:require('os') }
|
||||
* will allow the `os` module to be accessed in a Function node using:
|
||||
* global.get("os")
|
||||
*/
|
||||
functionGlobalContext: {
|
||||
// os:require('os'),
|
||||
},
|
||||
|
||||
/** The maximum number of messages nodes will buffer internally as part of their
|
||||
* operation. This applies across a range of nodes that operate on message sequences.
|
||||
* defaults to no limit. A value of 0 also means no limit is applied.
|
||||
*/
|
||||
//nodeMessageBufferMaxLength: 0,
|
||||
|
||||
/** If you installed the optional node-red-dashboard you can set it's path
|
||||
* relative to httpNodeRoot
|
||||
* Other optional properties include
|
||||
* readOnly:{boolean},
|
||||
* middleware:{function or array}, (req,res,next) - http middleware
|
||||
* ioMiddleware:{function or array}, (socket,next) - socket.io middleware
|
||||
*/
|
||||
//ui: { path: "ui" },
|
||||
|
||||
/** Colourise the console output of the debug node */
|
||||
//debugUseColors: true,
|
||||
|
||||
/** The maximum length, in characters, of any message sent to the debug sidebar tab */
|
||||
debugMaxLength: 1000,
|
||||
|
||||
/** Maximum buffer size for the exec node. Defaults to 10Mb */
|
||||
//execMaxBufferSize: 10000000,
|
||||
|
||||
/** Timeout in milliseconds for HTTP request connections. Defaults to 120s */
|
||||
//httpRequestTimeout: 120000,
|
||||
|
||||
/** Retry time in milliseconds for MQTT connections */
|
||||
mqttReconnectTime: 15000,
|
||||
|
||||
/** Retry time in milliseconds for Serial port connections */
|
||||
serialReconnectTime: 15000,
|
||||
|
||||
/** Retry time in milliseconds for TCP socket connections */
|
||||
//socketReconnectTime: 10000,
|
||||
|
||||
/** Timeout in milliseconds for TCP server socket connections. Defaults to no timeout */
|
||||
//socketTimeout: 120000,
|
||||
|
||||
/** Maximum number of messages to wait in queue while attempting to connect to TCP socket
|
||||
* defaults to 1000
|
||||
*/
|
||||
//tcpMsgQueueSize: 2000,
|
||||
|
||||
/** Timeout in milliseconds for inbound WebSocket connections that do not
|
||||
* match any configured node. Defaults to 5000
|
||||
*/
|
||||
//inboundWebSocketTimeout: 5000,
|
||||
|
||||
/** To disable the option for using local files for storing keys and
|
||||
* certificates in the TLS configuration node, set this to true.
|
||||
*/
|
||||
//tlsConfigDisableLocalFiles: true,
|
||||
|
||||
/** The following property can be used to verify websocket connection attempts.
|
||||
* This allows, for example, the HTTP request headers to be checked to ensure
|
||||
* they include valid authentication information.
|
||||
*/
|
||||
//webSocketNodeVerifyClient: function(info) {
|
||||
// /** 'info' has three properties:
|
||||
// * - origin : the value in the Origin header
|
||||
// * - req : the HTTP request
|
||||
// * - secure : true if req.connection.authorized or req.connection.encrypted is set
|
||||
// *
|
||||
// * The function should return true if the connection should be accepted, false otherwise.
|
||||
// *
|
||||
// * Alternatively, if this function is defined to accept a second argument, callback,
|
||||
// * it can be used to verify the client asynchronously.
|
||||
// * The callback takes three arguments:
|
||||
// * - result : boolean, whether to accept the connection or not
|
||||
// * - code : if result is false, the HTTP error status to return
|
||||
// * - reason: if result is false, the HTTP reason string to return
|
||||
// */
|
||||
//},
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,127 @@
|
|||
{
|
||||
"name": "node-red",
|
||||
"version": "2.2.3",
|
||||
"description": "Low-code programming for event-driven applications",
|
||||
"homepage": "http://nodered.org",
|
||||
"license": "Apache-2.0",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/node-red/node-red.git"
|
||||
},
|
||||
"private": "true",
|
||||
"scripts": {
|
||||
"start": "node $NODE_OPTIONS node_modules/node-red/red.js",
|
||||
"test": "grunt",
|
||||
"build": "grunt build",
|
||||
"dev": "grunt dev",
|
||||
"build-dev": "grunt build-dev",
|
||||
"docs": "grunt docs"
|
||||
},
|
||||
"contributors": [
|
||||
{
|
||||
"name": "Nick O'Leary"
|
||||
},
|
||||
{
|
||||
"name": "Dave Conway-Jones"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"acorn": "8.7.0",
|
||||
"acorn-walk": "8.2.0",
|
||||
"ajv": "8.10.0",
|
||||
"async-mutex": "0.3.2",
|
||||
"basic-auth": "2.0.1",
|
||||
"bcryptjs": "2.4.3",
|
||||
"body-parser": "1.19.1",
|
||||
"cheerio": "1.0.0-rc.10",
|
||||
"clone": "2.1.2",
|
||||
"content-type": "1.0.4",
|
||||
"cookie": "0.4.2",
|
||||
"cookie-parser": "1.4.6",
|
||||
"cors": "2.8.5",
|
||||
"cronosjs": "1.7.1",
|
||||
"denque": "2.0.1",
|
||||
"express": "4.17.2",
|
||||
"express-session": "1.17.2",
|
||||
"form-data": "4.0.0",
|
||||
"fs-extra": "10.0.0",
|
||||
"fs.notify": "0.0.4",
|
||||
"got": "11.8.3",
|
||||
"hash-sum": "2.0.0",
|
||||
"hpagent": "0.1.2",
|
||||
"https-proxy-agent": "5.0.0",
|
||||
"i18next": "21.6.11",
|
||||
"iconv-lite": "0.6.3",
|
||||
"is-utf8": "0.2.1",
|
||||
"js-yaml": "3.14.1",
|
||||
"json-stringify-safe": "5.0.1",
|
||||
"jsonata": "1.8.6",
|
||||
"lodash.clonedeep": "^4.5.0",
|
||||
"media-typer": "1.1.0",
|
||||
"memorystore": "1.6.7",
|
||||
"mime": "3.0.0",
|
||||
"moment-timezone": "0.5.34",
|
||||
"mqtt": "4.3.5",
|
||||
"multer": "1.4.4",
|
||||
"mustache": "4.2.0",
|
||||
"node-red": "2.2.3",
|
||||
"node-red-admin": "^2.2.3",
|
||||
"node-red-contrib-pythonshell": "github:namgk/node-red-contrib-pythonshell",
|
||||
"nopt": "5.0.0",
|
||||
"oauth2orize": "1.11.1",
|
||||
"on-headers": "1.0.2",
|
||||
"passport": "0.5.2",
|
||||
"passport-http-bearer": "1.0.1",
|
||||
"passport-oauth2-client-password": "0.1.2",
|
||||
"raw-body": "2.4.3",
|
||||
"semver": "7.3.5",
|
||||
"tar": "6.1.11",
|
||||
"tough-cookie": "4.0.0",
|
||||
"uglify-js": "3.15.1",
|
||||
"uuid": "8.3.2",
|
||||
"ws": "7.5.6",
|
||||
"xml2js": "0.4.23"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"bcrypt": "5.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"dompurify": "2.3.6",
|
||||
"grunt": "1.5.2",
|
||||
"grunt-chmod": "~1.1.1",
|
||||
"grunt-cli": "~1.4.3",
|
||||
"grunt-concurrent": "3.0.0",
|
||||
"grunt-contrib-clean": "~2.0.0",
|
||||
"grunt-contrib-compress": "2.0.0",
|
||||
"grunt-contrib-concat": "~1.0.1",
|
||||
"grunt-contrib-copy": "~1.0.0",
|
||||
"grunt-contrib-jshint": "3.1.1",
|
||||
"grunt-contrib-uglify": "5.0.1",
|
||||
"grunt-contrib-watch": "~1.1.0",
|
||||
"grunt-jsdoc": "2.4.1",
|
||||
"grunt-jsdoc-to-markdown": "6.0.0",
|
||||
"grunt-jsonlint": "2.1.3",
|
||||
"grunt-mkdir": "~1.1.0",
|
||||
"grunt-npm-command": "~0.1.2",
|
||||
"grunt-sass": "~3.1.0",
|
||||
"grunt-simple-mocha": "~0.4.1",
|
||||
"grunt-simple-nyc": "^3.0.1",
|
||||
"i18next-http-backend": "1.3.2",
|
||||
"jquery-i18next": "1.2.1",
|
||||
"jsdoc-nr-template": "github:node-red/jsdoc-nr-template",
|
||||
"marked": "4.0.12",
|
||||
"minami": "1.2.3",
|
||||
"mocha": "9.2.0",
|
||||
"node-red-node-test-helper": "^0.2.7",
|
||||
"nodemon": "2.0.15",
|
||||
"proxy": "^1.0.2",
|
||||
"sass": "1.49.7",
|
||||
"should": "13.2.3",
|
||||
"sinon": "11.1.2",
|
||||
"stoppable": "^1.1.0",
|
||||
"supertest": "6.2.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
target=$1
|
||||
|
||||
if [ "$#" -eq 1 ]
|
||||
then
|
||||
USERDIR=/data/$target
|
||||
|
||||
fi
|
||||
|
||||
echo "run with userdir=$USERDIR"
|
||||
|
||||
USERDIR=$USERDIR docker compose up
|
||||
#USERDIR=$USERDIR docker compose convert
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
#!/bin/bash
|
||||
|
||||
trap stop SIGINT SIGTERM
|
||||
|
||||
function stop() {
|
||||
kill $CHILD_PID
|
||||
wait $CHILD_PID
|
||||
}
|
||||
|
||||
/usr/local/bin/node $NODE_OPTIONS node_modules/node-red/red.js --userDir $USERDIR &
|
||||
|
||||
CHILD_PID="$!"
|
||||
|
||||
wait "${CHILD_PID}"
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
# RAG-on-kubeflow-with-nodered
|
||||
A simple example of RAG on kubeflow with node-red
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
git clone https://github.com/sefgsefg/RAG-pipeline-with-nodered.git
|
||||
```
|
||||
|
||||
```
|
||||
cd RAG_with_node-red/example
|
||||
```
|
||||
|
||||
```
|
||||
./run.sh main
|
||||
```
|
||||
|
||||
Problem solve: -bash: ./run.sh: Permission denied
|
||||
```
|
||||
chmod +x run.sh
|
||||
```
|
||||
|
||||
```
|
||||
cd scripts
|
||||
```
|
||||
|
||||
```
|
||||
chmod +x entrypoint.sh
|
||||
```
|
||||
|
||||
```
|
||||
cd ..
|
||||
```
|
||||
Run ./run.sh main again
|
||||
```
|
||||
./run.sh main
|
||||
```
|
||||
1.Insstall dependency and build the RAG flow on node-red.
|
||||
|
||||
|
||||
|
||||
2.Double click the RAG node and edit the infos.
|
||||
|
||||
|
||||
|
||||
3.Deploy and run the flow. It will run on the kubeflow pipeline.
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
kaggle
|
||||
pandas
|
||||
tqdm
|
||||
wget
|
||||
lightgbm
|
||||
pyarrow
|
||||
fastparquet
|
||||
kaggle
|
||||
pandas
|
||||
tqdm
|
||||
wget
|
||||
lightgbm
|
||||
pyarrow
|
||||
fastparquet
|
||||
|
|
|
|||
|
|
@ -1,348 +1,348 @@
|
|||
# Objective
|
||||
|
||||
This example is based on the Bluebook for bulldozers competition (https://www.kaggle.com/competitions/bluebook-for-bulldozers/overview). The objective of this exercise is to predict the sale price of bulldozers sold at auctions.
|
||||
|
||||
## Environment
|
||||
|
||||
This pipeline was tested using Kubeflow 1.4 and kfp 1.1.2 and x86-64 and ARM based system which includes all Intel and AMD based CPU's and M1/M2 series Macbooks.
|
||||
|
||||
## Step 1: Setup Kubeflow as a Service
|
||||
|
||||
- If you haven’t already, sign up (https://www.arrikto.com/kubeflow-as-a-service/)
|
||||
- Deploy Kubeflow
|
||||
|
||||
## Step 2: Launch a Notebook Server
|
||||
|
||||
- Bump memory to 2GB and vCPUs to 2
|
||||
|
||||
|
||||
## Step 3: Clone the Project Repo to Your Notebook
|
||||
|
||||
- (Kubeflow as a Service) Open up a terminal in the Notebook Server and git clone the kubeflow/examples repository
|
||||
```
|
||||
git clone https://github.com/kubeflow/examples
|
||||
```
|
||||
|
||||
## Step 4: Setup DockerHub and Docker
|
||||
|
||||
- If you haven’t already, sign up (https://hub.docker.com/) for DockerHub
|
||||
- If you haven’t already, install Docker Desktop (https://www.docker.com/products/docker-desktop/) locally OR install the Docker command line utility (https://docs.docker.com/get-docker/) and enter `sudo docker login` command in your terminal and log into Docker with your your DockerHub username and password
|
||||
|
||||
|
||||
## Step 5: Setup Kaggle
|
||||
|
||||
- If you haven’t already done so, sign up (https://www.kaggle.com/) for Kaggle
|
||||
- (On Kaggle) Generate an API token (https://www.kaggle.com/docs/api)
|
||||
- (Kubeflow as a Service) Create a Kubernetes secret
|
||||
```
|
||||
kubectl create secret generic kaggle-secret --from-literal=KAGGLE_USERNAME=<username> --from-literal=KAGGLE_KEY=<api_token>
|
||||
```
|
||||
|
||||
## Step 6: Install Git
|
||||
|
||||
- (Locally) If you don’t have it already, install Git (https://github.com/git-guides/install-git)
|
||||
|
||||
## Step 7: Clone the Project Repo Locally
|
||||
|
||||
- (Locally) Git clone the `kubeflow/examples` repository
|
||||
```
|
||||
git clone https://github.com/kubeflow/examples
|
||||
```
|
||||
|
||||
## Step 8: Create a PodDefault Resource
|
||||
|
||||
- (Kubeflow as a Service) Navigate to the `bluebook-for-bulldozers-kaggle-competition directory`
|
||||
- Create a resource.yaml file
|
||||
|
||||
resource.yaml:
|
||||
```
|
||||
apiVersion: "kubeflow.org/v1alpha1"
|
||||
kind: PodDefault
|
||||
metadata:
|
||||
name: kaggle-access
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
kaggle-secret: "true"
|
||||
desc: "kaggle-access"
|
||||
volumeMounts:
|
||||
- name: secret-volume
|
||||
mountPath: /secret/kaggle
|
||||
volumes:
|
||||
- name: secret-volume
|
||||
secret:
|
||||
secretName: kaggle-secret
|
||||
```
|
||||
|
||||
<img width="660" alt="image3" src="https://user-images.githubusercontent.com/17012391/177049116-b4186ec3-becb-40ea-973a-27bc52f90619.png">
|
||||
|
||||
- Apply resource.yaml using `kubectl apply -f resource.yaml`
|
||||
|
||||
## Step 9: Explore the load-data directory
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/load-data` directory
|
||||
- Open up the `load.py` file
|
||||
- Note the code in this file that will perform the actions required in the “load-data” pipeline step
|
||||
|
||||
<img width="209" alt="image7" src="https://user-images.githubusercontent.com/17012391/177049222-dab4c362-f06d-42ca-a07c-e61a0b301670.png">
|
||||
|
||||
## Step 10: Build the load Docker Image
|
||||
|
||||
- (Locally) Navigate to the bluebook-for-bulldozers-kaggle-competition/pipeline-components/load-data directory
|
||||
- Build the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker build --platform=linux/amd64 -t <docker_username>/<docker_imagename>:<tag>-amd64 .
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
|
||||
## Step 11: Push the load Docker Image to DockerHub
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/load-data` directory
|
||||
- Push the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>-amd64
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>
|
||||
```
|
||||
|
||||
## Step 12: Explore the preprocess directory
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/preprocess` directory
|
||||
- Open up the `preprocess.py` file
|
||||
- Note the code in this file that will perform the actions required in the “preprocess” pipeline step
|
||||
|
||||
<img width="209" alt="image5" src="https://user-images.githubusercontent.com/17012391/177049651-623fb24a-69c0-4a28-bbe1-3a360719b9ce.png">
|
||||
|
||||
## Step 13: Build the preprocess Docker Image
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/preprocess` directory
|
||||
- Build the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker build --platform=linux/amd64 -t <docker_username>/<docker_imagename>:<tag>-amd64 .
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
## Step 14: Push the preprocess Docker Image to DockerHub
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/preprocess` directory
|
||||
- Push the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>-amd64
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>
|
||||
```
|
||||
|
||||
## Step 15: Explore the train directory
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/train` directory
|
||||
- Open up the train.py file
|
||||
- Note the code in this file that will perform the actions required in the “train” pipeline step
|
||||
|
||||
|
||||
|
||||
|
||||
## Step 16: Build the train Docker Image
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/train` directory
|
||||
- Build the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker build --platform=linux/amd64 -t <docker_username>/<docker_imagename>:<tag>-amd64 .
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
|
||||
## Step 17: Push the train Docker Image to DockerHub
|
||||
|
||||
- (Locally) Navigate to the bluebook-for-bulldozers-kaggle-competition/pipeline-components/train directory
|
||||
- Push the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>-amd64
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>
|
||||
```
|
||||
|
||||
## Step 18: Explore the test directory
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/test` directory
|
||||
- Open up the `test.py` file
|
||||
- Note the code in this file that will perform the actions required in the “test” pipeline step
|
||||
|
||||
<img width="237" alt="image6" src="https://user-images.githubusercontent.com/17012391/177051308-31b27ad7-71f3-47af-a068-de381fe64b5b.png">
|
||||
|
||||
## Step 19: Build the test Docker Image
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/test` directory
|
||||
- Build the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker build --platform=linux/amd64 -t <docker_username>/<docker_imagename>:<tag>-amd64 .
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
## Step 20: Push the test Docker Image to DockerHub
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/test` directory
|
||||
- Push the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>-amd64
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>
|
||||
```
|
||||
|
||||
## Step 21: Modify the blue-book-for-bulldozers-kfp.py file
|
||||
|
||||
(Kubeflow as a Service) Navigate to the `bluebook-for-bulldozers-kaggle-competition` directory
|
||||
Update the `bluebook-for-bulldozers-kaggle-competition-kfp.py` with accurate Docker Image inputs
|
||||
|
||||
```
|
||||
return dsl.ContainerOp(
|
||||
name = 'load-data',
|
||||
image = '<dockerhub username>/<image name>:<tag>',
|
||||
|
||||
—-----
|
||||
|
||||
def PreProcess(comp1):
|
||||
return dsl.ContainerOp(
|
||||
name = 'preprocess',
|
||||
image = '<dockerhub username>/<image name>:<tag>',
|
||||
|
||||
—-----
|
||||
|
||||
def Train(comp2):
|
||||
return dsl.ContainerOp(
|
||||
name = 'train',
|
||||
image = '<dockerhub username>/<image name>:<tag>',
|
||||
|
||||
—-----
|
||||
|
||||
def Test(comp3):
|
||||
return dsl.ContainerOp(
|
||||
name = 'test',
|
||||
image = '<dockerhub username>/<image name>:<tag>',
|
||||
|
||||
```
|
||||
|
||||
## Step 22: Generate a KFP Pipeline yaml File
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition` directory and delete the existing `blue-book-for-bulldozers-kaggle-competition-kfp.yaml` file
|
||||
- (Kubeflow as a Service) Navigate to the `bluebook-for-bulldozers-kaggle-competition` directory
|
||||
|
||||
Build a python virtual environment:
|
||||
|
||||
Step a) Update pip
|
||||
```
|
||||
python3 -m pip install --upgrade pip
|
||||
```
|
||||
|
||||
Step b) Install virtualenv
|
||||
```
|
||||
sudo pip3 install virtualenv
|
||||
```
|
||||
|
||||
Step c) Check the installed version of venv
|
||||
```
|
||||
virtualenv --version
|
||||
```
|
||||
|
||||
Step d) Name your virtual enviornment as kfp
|
||||
```
|
||||
virtualenv kfp
|
||||
```
|
||||
|
||||
Step e) Activate your venv.
|
||||
```
|
||||
source kfp/bin/activate
|
||||
```
|
||||
|
||||
After this virtual environment will get activated. Now in our activated venv we need to install following packages:
|
||||
```
|
||||
sudo apt-get update
|
||||
sudo apt-get upgrade
|
||||
sudo apt-get install -y git python3-pip
|
||||
|
||||
python3 -m pip install kfp==1.1.2
|
||||
```
|
||||
|
||||
After installing packages create the yaml file
|
||||
|
||||
Inside venv point your terminal to a path which contains our kfp file to build pipeline (blue-book-for-bulldozers-kaggle-competition-kfp.py) and run these commands to generate a `yaml` file for the Pipeline:
|
||||
|
||||
```
|
||||
blue-book-for-bulldozers-kaggle-competition-kfp.py
|
||||
```
|
||||
|
||||
<img width="496" alt="Screenshot 2022-07-04 at 12 01 51 AM" src="https://user-images.githubusercontent.com/17012391/177052742-4dae4647-b51b-4f36-94b0-3114130c756b.png">
|
||||
|
||||
- Download the `bluebook-for-bulldozers-kaggle-competition.yaml` file that was created to your local `bluebook-for-bulldozers-kaggle-competition` directory
|
||||
|
||||
## Step 23: Create an Experiment
|
||||
|
||||
- (Kubeflow as a Service) Within the Kubeflow Central Dashboard, navigate to the Experiments (KFP) > Create Experiment view
|
||||
- Name the experiment and click Next
|
||||
- Click on Experiments (KFP) to view the experiment you just created
|
||||
|
||||
## Step 24: Create a Pipeline
|
||||
|
||||
- (Kubeflow as a Service) Within the Kubeflow Central Dashboard, navigate to the Pipelines > +Upload Pipeline view
|
||||
- Name the pipeline
|
||||
- Click on Upload a file
|
||||
- Upload the local bluebook-for-bulldozers-kaggle-competition.py.yaml file
|
||||
- Click Create
|
||||
|
||||
Step 25: Create a Run
|
||||
|
||||
- (Kubeflow as a Service) Click on Create Run in the view from the previous step
|
||||
- Choose the experiment we created in Step 23
|
||||
- Click Start
|
||||
- Click on the run name to view the runtime execution graph
|
||||
|
||||
<img width="285" alt="Screenshot 2022-07-04 at 12 04 43 AM" src="https://user-images.githubusercontent.com/17012391/177052835-d2cde097-7354-4cac-8876-7d89244864d3.png">
|
||||
|
||||
|
||||
## Troubleshooting Tips:
|
||||
While running the pipeline as mentioned above you may come across this error:
|
||||
|
||||
|
||||
errorlog:
|
||||
|
||||
```
|
||||
kaggle.rest.ApiException: (403)
|
||||
Reason: Forbidden
|
||||
HTTP response headers: HTTPHeaderDict({'Content-Type': 'application/json', 'Date': 'Thu, 23 Jun 2022 11:31:18 GMT', 'Access-Control-Allow-Credentials': 'true', 'Set-Cookie': 'ka_sessionid=6817a347c75399a531148e19cad0aaeb; max-age=2626560; path=/, GCLB=CIGths3--ebbUg; path=/; HttpOnly', 'Transfer-Encoding': 'chunked', 'Vary':
|
||||
HTTP response body: b'{"code":403,"message":"You must accept this competition\\u0027s rules before you\\u0027ll be able to download files."}'
|
||||
|
||||
```
|
||||
This error occours for two reasons:
|
||||
- Your Kaggle account is not verified with your phone number.
|
||||
- Rules for this specific competitions are not accepted.
|
||||
|
||||
Lets accept Rules of Bulldozers competition
|
||||
|
||||
|
||||
Click on "I Understand and Accept". After this you will be prompted to verify your account using your phone number:
|
||||
|
||||
|
||||
Add your phone number and Kaggle will send the code to your number, enter this code and verify your account. ( Note: pipeline wont run if your Kaggle account is not verified )
|
||||
|
||||
## Success
|
||||
After the kaggle account is verified pipeline run is successful we will get the following:
|
||||
|
||||
<img width="1380" alt="Screenshot 2022-06-10 at 12 04 48 AM" src="https://user-images.githubusercontent.com/17012391/172920115-ccefd333-c500-4577-afcb-8487c2208371.png">
|
||||
|
||||
|
||||
# Objective
|
||||
|
||||
This example is based on the Bluebook for bulldozers competition (https://www.kaggle.com/competitions/bluebook-for-bulldozers/overview). The objective of this exercise is to predict the sale price of bulldozers sold at auctions.
|
||||
|
||||
## Environment
|
||||
|
||||
This pipeline was tested using Kubeflow 1.4 and kfp 1.1.2 and x86-64 and ARM based system which includes all Intel and AMD based CPU's and M1/M2 series Macbooks.
|
||||
|
||||
## Step 1: Setup Kubeflow as a Service
|
||||
|
||||
- If you haven’t already, sign up (https://www.arrikto.com/kubeflow-as-a-service/)
|
||||
- Deploy Kubeflow
|
||||
|
||||
## Step 2: Launch a Notebook Server
|
||||
|
||||
- Bump memory to 2GB and vCPUs to 2
|
||||
|
||||
|
||||
## Step 3: Clone the Project Repo to Your Notebook
|
||||
|
||||
- (Kubeflow as a Service) Open up a terminal in the Notebook Server and git clone the kubeflow/examples repository
|
||||
```
|
||||
git clone https://github.com/kubeflow/examples
|
||||
```
|
||||
|
||||
## Step 4: Setup DockerHub and Docker
|
||||
|
||||
- If you haven’t already, sign up (https://hub.docker.com/) for DockerHub
|
||||
- If you haven’t already, install Docker Desktop (https://www.docker.com/products/docker-desktop/) locally OR install the Docker command line utility (https://docs.docker.com/get-docker/) and enter `sudo docker login` command in your terminal and log into Docker with your your DockerHub username and password
|
||||
|
||||
|
||||
## Step 5: Setup Kaggle
|
||||
|
||||
- If you haven’t already done so, sign up (https://www.kaggle.com/) for Kaggle
|
||||
- (On Kaggle) Generate an API token (https://www.kaggle.com/docs/api)
|
||||
- (Kubeflow as a Service) Create a Kubernetes secret
|
||||
```
|
||||
kubectl create secret generic kaggle-secret --from-literal=KAGGLE_USERNAME=<username> --from-literal=KAGGLE_KEY=<api_token>
|
||||
```
|
||||
|
||||
## Step 6: Install Git
|
||||
|
||||
- (Locally) If you don’t have it already, install Git (https://github.com/git-guides/install-git)
|
||||
|
||||
## Step 7: Clone the Project Repo Locally
|
||||
|
||||
- (Locally) Git clone the `kubeflow/examples` repository
|
||||
```
|
||||
git clone https://github.com/kubeflow/examples
|
||||
```
|
||||
|
||||
## Step 8: Create a PodDefault Resource
|
||||
|
||||
- (Kubeflow as a Service) Navigate to the `bluebook-for-bulldozers-kaggle-competition directory`
|
||||
- Create a resource.yaml file
|
||||
|
||||
resource.yaml:
|
||||
```
|
||||
apiVersion: "kubeflow.org/v1alpha1"
|
||||
kind: PodDefault
|
||||
metadata:
|
||||
name: kaggle-access
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
kaggle-secret: "true"
|
||||
desc: "kaggle-access"
|
||||
volumeMounts:
|
||||
- name: secret-volume
|
||||
mountPath: /secret/kaggle
|
||||
volumes:
|
||||
- name: secret-volume
|
||||
secret:
|
||||
secretName: kaggle-secret
|
||||
```
|
||||
|
||||
<img width="660" alt="image3" src="https://user-images.githubusercontent.com/17012391/177049116-b4186ec3-becb-40ea-973a-27bc52f90619.png">
|
||||
|
||||
- Apply resource.yaml using `kubectl apply -f resource.yaml`
|
||||
|
||||
## Step 9: Explore the load-data directory
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/load-data` directory
|
||||
- Open up the `load.py` file
|
||||
- Note the code in this file that will perform the actions required in the “load-data” pipeline step
|
||||
|
||||
<img width="209" alt="image7" src="https://user-images.githubusercontent.com/17012391/177049222-dab4c362-f06d-42ca-a07c-e61a0b301670.png">
|
||||
|
||||
## Step 10: Build the load Docker Image
|
||||
|
||||
- (Locally) Navigate to the bluebook-for-bulldozers-kaggle-competition/pipeline-components/load-data directory
|
||||
- Build the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker build --platform=linux/amd64 -t <docker_username>/<docker_imagename>:<tag>-amd64 .
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
|
||||
## Step 11: Push the load Docker Image to DockerHub
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/load-data` directory
|
||||
- Push the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>-amd64
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>
|
||||
```
|
||||
|
||||
## Step 12: Explore the preprocess directory
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/preprocess` directory
|
||||
- Open up the `preprocess.py` file
|
||||
- Note the code in this file that will perform the actions required in the “preprocess” pipeline step
|
||||
|
||||
<img width="209" alt="image5" src="https://user-images.githubusercontent.com/17012391/177049651-623fb24a-69c0-4a28-bbe1-3a360719b9ce.png">
|
||||
|
||||
## Step 13: Build the preprocess Docker Image
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/preprocess` directory
|
||||
- Build the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker build --platform=linux/amd64 -t <docker_username>/<docker_imagename>:<tag>-amd64 .
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
## Step 14: Push the preprocess Docker Image to DockerHub
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/preprocess` directory
|
||||
- Push the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>-amd64
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>
|
||||
```
|
||||
|
||||
## Step 15: Explore the train directory
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/train` directory
|
||||
- Open up the train.py file
|
||||
- Note the code in this file that will perform the actions required in the “train” pipeline step
|
||||
|
||||
|
||||
|
||||
|
||||
## Step 16: Build the train Docker Image
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/train` directory
|
||||
- Build the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker build --platform=linux/amd64 -t <docker_username>/<docker_imagename>:<tag>-amd64 .
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
|
||||
## Step 17: Push the train Docker Image to DockerHub
|
||||
|
||||
- (Locally) Navigate to the bluebook-for-bulldozers-kaggle-competition/pipeline-components/train directory
|
||||
- Push the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>-amd64
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>
|
||||
```
|
||||
|
||||
## Step 18: Explore the test directory
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/test` directory
|
||||
- Open up the `test.py` file
|
||||
- Note the code in this file that will perform the actions required in the “test” pipeline step
|
||||
|
||||
<img width="237" alt="image6" src="https://user-images.githubusercontent.com/17012391/177051308-31b27ad7-71f3-47af-a068-de381fe64b5b.png">
|
||||
|
||||
## Step 19: Build the test Docker Image
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/test` directory
|
||||
- Build the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker build --platform=linux/amd64 -t <docker_username>/<docker_imagename>:<tag>-amd64 .
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker build -t <docker_username>/<docker_imagename>:<tag> .
|
||||
```
|
||||
## Step 20: Push the test Docker Image to DockerHub
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition/pipeline-components/test` directory
|
||||
- Push the Docker image if locally you are using arm64 (Apple M1)
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>-amd64
|
||||
```
|
||||
- OR build the Docker image if locally you are using amd64
|
||||
```
|
||||
docker push <docker_username>/<docker_imagename>:<tag>
|
||||
```
|
||||
|
||||
## Step 21: Modify the blue-book-for-bulldozers-kfp.py file
|
||||
|
||||
(Kubeflow as a Service) Navigate to the `bluebook-for-bulldozers-kaggle-competition` directory
|
||||
Update the `bluebook-for-bulldozers-kaggle-competition-kfp.py` with accurate Docker Image inputs
|
||||
|
||||
```
|
||||
return dsl.ContainerOp(
|
||||
name = 'load-data',
|
||||
image = '<dockerhub username>/<image name>:<tag>',
|
||||
|
||||
—-----
|
||||
|
||||
def PreProcess(comp1):
|
||||
return dsl.ContainerOp(
|
||||
name = 'preprocess',
|
||||
image = '<dockerhub username>/<image name>:<tag>',
|
||||
|
||||
—-----
|
||||
|
||||
def Train(comp2):
|
||||
return dsl.ContainerOp(
|
||||
name = 'train',
|
||||
image = '<dockerhub username>/<image name>:<tag>',
|
||||
|
||||
—-----
|
||||
|
||||
def Test(comp3):
|
||||
return dsl.ContainerOp(
|
||||
name = 'test',
|
||||
image = '<dockerhub username>/<image name>:<tag>',
|
||||
|
||||
```
|
||||
|
||||
## Step 22: Generate a KFP Pipeline yaml File
|
||||
|
||||
- (Locally) Navigate to the `bluebook-for-bulldozers-kaggle-competition` directory and delete the existing `blue-book-for-bulldozers-kaggle-competition-kfp.yaml` file
|
||||
- (Kubeflow as a Service) Navigate to the `bluebook-for-bulldozers-kaggle-competition` directory
|
||||
|
||||
Build a python virtual environment:
|
||||
|
||||
Step a) Update pip
|
||||
```
|
||||
python3 -m pip install --upgrade pip
|
||||
```
|
||||
|
||||
Step b) Install virtualenv
|
||||
```
|
||||
sudo pip3 install virtualenv
|
||||
```
|
||||
|
||||
Step c) Check the installed version of venv
|
||||
```
|
||||
virtualenv --version
|
||||
```
|
||||
|
||||
Step d) Name your virtual enviornment as kfp
|
||||
```
|
||||
virtualenv kfp
|
||||
```
|
||||
|
||||
Step e) Activate your venv.
|
||||
```
|
||||
source kfp/bin/activate
|
||||
```
|
||||
|
||||
After this virtual environment will get activated. Now in our activated venv we need to install following packages:
|
||||
```
|
||||
sudo apt-get update
|
||||
sudo apt-get upgrade
|
||||
sudo apt-get install -y git python3-pip
|
||||
|
||||
python3 -m pip install kfp==1.1.2
|
||||
```
|
||||
|
||||
After installing packages create the yaml file
|
||||
|
||||
Inside venv point your terminal to a path which contains our kfp file to build pipeline (blue-book-for-bulldozers-kaggle-competition-kfp.py) and run these commands to generate a `yaml` file for the Pipeline:
|
||||
|
||||
```
|
||||
blue-book-for-bulldozers-kaggle-competition-kfp.py
|
||||
```
|
||||
|
||||
<img width="496" alt="Screenshot 2022-07-04 at 12 01 51 AM" src="https://user-images.githubusercontent.com/17012391/177052742-4dae4647-b51b-4f36-94b0-3114130c756b.png">
|
||||
|
||||
- Download the `bluebook-for-bulldozers-kaggle-competition.yaml` file that was created to your local `bluebook-for-bulldozers-kaggle-competition` directory
|
||||
|
||||
## Step 23: Create an Experiment
|
||||
|
||||
- (Kubeflow as a Service) Within the Kubeflow Central Dashboard, navigate to the Experiments (KFP) > Create Experiment view
|
||||
- Name the experiment and click Next
|
||||
- Click on Experiments (KFP) to view the experiment you just created
|
||||
|
||||
## Step 24: Create a Pipeline
|
||||
|
||||
- (Kubeflow as a Service) Within the Kubeflow Central Dashboard, navigate to the Pipelines > +Upload Pipeline view
|
||||
- Name the pipeline
|
||||
- Click on Upload a file
|
||||
- Upload the local bluebook-for-bulldozers-kaggle-competition.py.yaml file
|
||||
- Click Create
|
||||
|
||||
Step 25: Create a Run
|
||||
|
||||
- (Kubeflow as a Service) Click on Create Run in the view from the previous step
|
||||
- Choose the experiment we created in Step 23
|
||||
- Click Start
|
||||
- Click on the run name to view the runtime execution graph
|
||||
|
||||
<img width="285" alt="Screenshot 2022-07-04 at 12 04 43 AM" src="https://user-images.githubusercontent.com/17012391/177052835-d2cde097-7354-4cac-8876-7d89244864d3.png">
|
||||
|
||||
|
||||
## Troubleshooting Tips:
|
||||
While running the pipeline as mentioned above you may come across this error:
|
||||
|
||||
|
||||
errorlog:
|
||||
|
||||
```
|
||||
kaggle.rest.ApiException: (403)
|
||||
Reason: Forbidden
|
||||
HTTP response headers: HTTPHeaderDict({'Content-Type': 'application/json', 'Date': 'Thu, 23 Jun 2022 11:31:18 GMT', 'Access-Control-Allow-Credentials': 'true', 'Set-Cookie': 'ka_sessionid=6817a347c75399a531148e19cad0aaeb; max-age=2626560; path=/, GCLB=CIGths3--ebbUg; path=/; HttpOnly', 'Transfer-Encoding': 'chunked', 'Vary':
|
||||
HTTP response body: b'{"code":403,"message":"You must accept this competition\\u0027s rules before you\\u0027ll be able to download files."}'
|
||||
|
||||
```
|
||||
This error occours for two reasons:
|
||||
- Your Kaggle account is not verified with your phone number.
|
||||
- Rules for this specific competitions are not accepted.
|
||||
|
||||
Lets accept Rules of Bulldozers competition
|
||||
|
||||
|
||||
Click on "I Understand and Accept". After this you will be prompted to verify your account using your phone number:
|
||||
|
||||
|
||||
Add your phone number and Kaggle will send the code to your number, enter this code and verify your account. ( Note: pipeline wont run if your Kaggle account is not verified )
|
||||
|
||||
## Success
|
||||
After the kaggle account is verified pipeline run is successful we will get the following:
|
||||
|
||||
<img width="1380" alt="Screenshot 2022-06-10 at 12 04 48 AM" src="https://user-images.githubusercontent.com/17012391/172920115-ccefd333-c500-4577-afcb-8487c2208371.png">
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,62 +1,62 @@
|
|||
import kfp
|
||||
from kfp import dsl
|
||||
|
||||
def LoadData():
|
||||
vop = dsl.VolumeOp(name="pvc",
|
||||
resource_name="pvc", size='1Gi',
|
||||
modes=dsl.VOLUME_MODE_RWO)
|
||||
|
||||
return dsl.ContainerOp(
|
||||
name = 'load-data',
|
||||
image = 'hubdocker76/bulldozers:v6',
|
||||
command = ['python3', 'load.py'],
|
||||
|
||||
pvolumes={
|
||||
'/data': vop.volume
|
||||
}
|
||||
)
|
||||
|
||||
def PreProcess(comp1):
|
||||
return dsl.ContainerOp(
|
||||
name = 'preprocess',
|
||||
image = 'hubdocker76/bulldozers-preprocess:v1',
|
||||
pvolumes={
|
||||
'/data': comp1.pvolumes['/data']
|
||||
},
|
||||
command = ['python3', 'preprocess.py']
|
||||
)
|
||||
|
||||
def Train(comp2):
|
||||
return dsl.ContainerOp(
|
||||
name = 'train',
|
||||
image = 'hubdocker76/bulldozers-train:v2',
|
||||
pvolumes={
|
||||
'/data': comp2.pvolumes['/data']
|
||||
},
|
||||
command = ['python3', 'train.py']
|
||||
)
|
||||
|
||||
def Test(comp3):
|
||||
return dsl.ContainerOp(
|
||||
name = 'test',
|
||||
image = 'hubdocker76/bulldozers-test:v2',
|
||||
pvolumes={
|
||||
'/data': comp3.pvolumes['/data']
|
||||
},
|
||||
command = ['python3', 'test.py']
|
||||
)
|
||||
|
||||
|
||||
@dsl.pipeline(
|
||||
name = 'blue book for bulldozers',
|
||||
description = 'pipeline to run blue book for bulldozers')
|
||||
|
||||
def passing_parameter():
|
||||
comp1 = LoadData().add_pod_label("kaggle-secret", "true")
|
||||
comp2 = PreProcess(comp1)
|
||||
comp3 = Train(comp2)
|
||||
comp4 = Test(comp3)
|
||||
|
||||
if __name__ == '__main__':
|
||||
import kfp.compiler as compiler
|
||||
compiler.Compiler().compile(passing_parameter, __file__[:-3]+ '.yaml')
|
||||
import kfp
|
||||
from kfp import dsl
|
||||
|
||||
def LoadData():
|
||||
vop = dsl.VolumeOp(name="pvc",
|
||||
resource_name="pvc", size='1Gi',
|
||||
modes=dsl.VOLUME_MODE_RWO)
|
||||
|
||||
return dsl.ContainerOp(
|
||||
name = 'load-data',
|
||||
image = 'hubdocker76/bulldozers:v6',
|
||||
command = ['python3', 'load.py'],
|
||||
|
||||
pvolumes={
|
||||
'/data': vop.volume
|
||||
}
|
||||
)
|
||||
|
||||
def PreProcess(comp1):
|
||||
return dsl.ContainerOp(
|
||||
name = 'preprocess',
|
||||
image = 'hubdocker76/bulldozers-preprocess:v1',
|
||||
pvolumes={
|
||||
'/data': comp1.pvolumes['/data']
|
||||
},
|
||||
command = ['python3', 'preprocess.py']
|
||||
)
|
||||
|
||||
def Train(comp2):
|
||||
return dsl.ContainerOp(
|
||||
name = 'train',
|
||||
image = 'hubdocker76/bulldozers-train:v2',
|
||||
pvolumes={
|
||||
'/data': comp2.pvolumes['/data']
|
||||
},
|
||||
command = ['python3', 'train.py']
|
||||
)
|
||||
|
||||
def Test(comp3):
|
||||
return dsl.ContainerOp(
|
||||
name = 'test',
|
||||
image = 'hubdocker76/bulldozers-test:v2',
|
||||
pvolumes={
|
||||
'/data': comp3.pvolumes['/data']
|
||||
},
|
||||
command = ['python3', 'test.py']
|
||||
)
|
||||
|
||||
|
||||
@dsl.pipeline(
|
||||
name = 'blue book for bulldozers',
|
||||
description = 'pipeline to run blue book for bulldozers')
|
||||
|
||||
def passing_parameter():
|
||||
comp1 = LoadData().add_pod_label("kaggle-secret", "true")
|
||||
comp2 = PreProcess(comp1)
|
||||
comp3 = Train(comp2)
|
||||
comp4 = Test(comp3)
|
||||
|
||||
if __name__ == '__main__':
|
||||
import kfp.compiler as compiler
|
||||
compiler.Compiler().compile(passing_parameter, __file__[:-3]+ '.yaml')
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
local components = std.extVar("__ksonnet/components");
|
||||
components + {
|
||||
// Insert user-specified overrides here.
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
{
|
||||
// Warning: Do not define a global "image" as that will end up overriding
|
||||
// the image parameter for all components. Define more specific names
|
||||
// e.g. "dataflowImage", "trainerCpuImage", "trainerGpuImage",
|
||||
workingDir: "gs://code-search-demo/20181104",
|
||||
dataDir: "gs://code-search-demo/20181104/data",
|
||||
project: "code-search-demo",
|
||||
experiment: "demo-trainer-11-07-dist-sync-gpu",
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
local base = import "base.libsonnet";
|
||||
// uncomment if you reference ksonnet-lib
|
||||
// local k = import "k.libsonnet";
|
||||
|
||||
base {
|
||||
// Insert user-specified overrides here. For example if a component is named \"nginx-deployment\", you might have something like:\n")
|
||||
// "nginx-deployment"+: k.deployment.mixin.metadata.labels({foo: "bar"})
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
local params = std.extVar('__ksonnet/params');
|
||||
local globals = import 'globals.libsonnet';
|
||||
local envParams = params {
|
||||
components+: {
|
||||
"t2t-code-search"+: {},
|
||||
"t2t-code-search-datagen"+: {
|
||||
githubTable: '',
|
||||
},
|
||||
"submit-preprocess-job"+: {
|
||||
githubTable: '',
|
||||
},
|
||||
"search-index-server"+: {
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
{
|
||||
components: {
|
||||
[x]: envParams.components[x] + globals
|
||||
for x in std.objectFields(envParams.components)
|
||||
},
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
{
|
||||
// Warning: Do not define a global "image" as that will end up overriding
|
||||
// the image parameter for all components. Define more specific names
|
||||
// e.g. "dataflowImage", "trainerCpuImage", "trainerGpuImage",
|
||||
experiment: "pipeline",
|
||||
waitUntilFinish: "true",
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
local base = import "base.libsonnet";
|
||||
// uncomment if you reference ksonnet-lib
|
||||
// local k = import "k.libsonnet";
|
||||
|
||||
base + {
|
||||
// Insert user-specified overrides here. For example if a component is named \"nginx-deployment\", you might have something like:\n")
|
||||
// "nginx-deployment"+: k.deployment.mixin.metadata.labels({foo: "bar"})
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
local params = std.extVar("__ksonnet/params");
|
||||
local globals = import "globals.libsonnet";
|
||||
local envParams = params + {
|
||||
components +: {
|
||||
},
|
||||
};
|
||||
|
||||
{
|
||||
components: {
|
||||
[x]: envParams.components[x] + globals, for x in std.objectFields(envParams.components)
|
||||
},
|
||||
}
|
||||
|
|
@ -1,62 +1,62 @@
|
|||
|
||||
import kfp
|
||||
from kfp import dsl
|
||||
|
||||
def create_pv():
|
||||
return dsl.VolumeOp(
|
||||
name="create_pv",
|
||||
resource_name="kfp-pvc",
|
||||
size="1Gi",
|
||||
modes=dsl.VOLUME_MODE_RWO
|
||||
)
|
||||
|
||||
|
||||
def parallel_1(vol_name: str):
|
||||
cop = dsl.ContainerOp(
|
||||
name='generate_data',
|
||||
image='bash:5.1',
|
||||
command=['sh', '-c'],
|
||||
arguments=['echo 1 | tee /mnt/out1.txt']
|
||||
)
|
||||
cop.container.set_image_pull_policy('IfNotPresent')
|
||||
cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)})
|
||||
return cop
|
||||
|
||||
|
||||
def parallel_2(vol_name: str):
|
||||
cop = dsl.ContainerOp(
|
||||
name='generate_data',
|
||||
image='bash:5.1',
|
||||
command=['sh', '-c'],
|
||||
arguments=['echo 2 | tee /mnt/out2.txt']
|
||||
)
|
||||
cop.container.set_image_pull_policy('IfNotPresent')
|
||||
cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)})
|
||||
return cop
|
||||
|
||||
|
||||
def parallel_3(vol_name: str):
|
||||
cop = dsl.ContainerOp(
|
||||
name='generate_data',
|
||||
image='bash:5.1',
|
||||
command=['sh', '-c'],
|
||||
arguments=['echo 3 | tee /mnt/out3.txt']
|
||||
)
|
||||
cop.container.set_image_pull_policy('IfNotPresent')
|
||||
cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)})
|
||||
return cop
|
||||
|
||||
|
||||
@dsl.pipeline(
|
||||
name="Kubeflow volume parallel example",
|
||||
description="Demonstrate the use case of volume on Kubeflow pipeline.")
|
||||
def volume_parallel():
|
||||
vop = create_pv()
|
||||
cop1 = parallel_1(vop.outputs["name"]).after(vop)
|
||||
cop2 = parallel_2(vop.outputs["name"]).after(vop)
|
||||
cop3 = parallel_3(vop.outputs["name"]).after(vop)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import kfp.compiler as compiler
|
||||
|
||||
import kfp
|
||||
from kfp import dsl
|
||||
|
||||
def create_pv():
|
||||
return dsl.VolumeOp(
|
||||
name="create_pv",
|
||||
resource_name="kfp-pvc",
|
||||
size="1Gi",
|
||||
modes=dsl.VOLUME_MODE_RWO
|
||||
)
|
||||
|
||||
|
||||
def parallel_1(vol_name: str):
|
||||
cop = dsl.ContainerOp(
|
||||
name='generate_data',
|
||||
image='bash:5.1',
|
||||
command=['sh', '-c'],
|
||||
arguments=['echo 1 | tee /mnt/out1.txt']
|
||||
)
|
||||
cop.container.set_image_pull_policy('IfNotPresent')
|
||||
cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)})
|
||||
return cop
|
||||
|
||||
|
||||
def parallel_2(vol_name: str):
|
||||
cop = dsl.ContainerOp(
|
||||
name='generate_data',
|
||||
image='bash:5.1',
|
||||
command=['sh', '-c'],
|
||||
arguments=['echo 2 | tee /mnt/out2.txt']
|
||||
)
|
||||
cop.container.set_image_pull_policy('IfNotPresent')
|
||||
cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)})
|
||||
return cop
|
||||
|
||||
|
||||
def parallel_3(vol_name: str):
|
||||
cop = dsl.ContainerOp(
|
||||
name='generate_data',
|
||||
image='bash:5.1',
|
||||
command=['sh', '-c'],
|
||||
arguments=['echo 3 | tee /mnt/out3.txt']
|
||||
)
|
||||
cop.container.set_image_pull_policy('IfNotPresent')
|
||||
cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)})
|
||||
return cop
|
||||
|
||||
|
||||
@dsl.pipeline(
|
||||
name="Kubeflow volume parallel example",
|
||||
description="Demonstrate the use case of volume on Kubeflow pipeline.")
|
||||
def volume_parallel():
|
||||
vop = create_pv()
|
||||
cop1 = parallel_1(vop.outputs["name"]).after(vop)
|
||||
cop2 = parallel_2(vop.outputs["name"]).after(vop)
|
||||
cop3 = parallel_3(vop.outputs["name"]).after(vop)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import kfp.compiler as compiler
|
||||
compiler.Compiler().compile(volume_parallel, __file__ + ".yaml")
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,4 +1,4 @@
|
|||
pandas==1.1.5
|
||||
seaborn==0.9.0
|
||||
tensorflow==2.3.0
|
||||
wget==3.2
|
||||
pandas==1.1.5
|
||||
seaborn==0.9.0
|
||||
tensorflow==2.3.0
|
||||
wget==3.2
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue