mirror of https://github.com/kubeflow/examples.git
958 lines
50 KiB
Plaintext
958 lines
50 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 30,
|
|
"id": "69637b5f",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Requirement already satisfied: kfp==1.4.0 in ./.local/lib/python3.8/site-packages (from -r requirement.txt (line 1)) (1.4.0)\n",
|
|
"Requirement already satisfied: numpy in ./.local/lib/python3.8/site-packages (from -r requirement.txt (line 2)) (1.21.2)\n",
|
|
"Requirement already satisfied: keras in ./.local/lib/python3.8/site-packages (from -r requirement.txt (line 3)) (2.6.0)\n",
|
|
"Requirement already satisfied: tqdm in ./.local/lib/python3.8/site-packages (from -r requirement.txt (line 4)) (4.62.3)\n",
|
|
"Requirement already satisfied: config in ./.local/lib/python3.8/site-packages (from -r requirement.txt (line 5)) (0.5.1)\n",
|
|
"Requirement already satisfied: sklearn in ./.local/lib/python3.8/site-packages (from -r requirement.txt (line 6)) (0.0)\n",
|
|
"Requirement already satisfied: kubernetes<12.0.0,>=8.0.0 in ./.local/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (11.0.0)\n",
|
|
"Requirement already satisfied: Deprecated in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (1.2.12)\n",
|
|
"Requirement already satisfied: kfp-pipeline-spec<0.2.0,>=0.1.0 in ./.local/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (0.1.9)\n",
|
|
"Requirement already satisfied: requests-toolbelt>=0.8.0 in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (0.9.1)\n",
|
|
"Requirement already satisfied: kfp-server-api<2.0.0,>=1.1.2 in ./.local/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (1.6.0)\n",
|
|
"Requirement already satisfied: docstring-parser>=0.7.3 in ./.local/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (0.10)\n",
|
|
"Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (5.4.1)\n",
|
|
"Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (7.1.2)\n",
|
|
"Requirement already satisfied: jsonschema>=3.0.1 in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (3.2.0)\n",
|
|
"Requirement already satisfied: google-cloud-storage>=1.13.0 in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (1.37.1)\n",
|
|
"Requirement already satisfied: cloudpickle in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (1.6.0)\n",
|
|
"Requirement already satisfied: fire>=0.3.1 in ./.local/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (0.4.0)\n",
|
|
"Requirement already satisfied: tabulate in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (0.8.9)\n",
|
|
"Requirement already satisfied: strip-hints in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (0.1.9)\n",
|
|
"Requirement already satisfied: google-auth>=1.6.1 in /opt/conda/lib/python3.8/site-packages (from kfp==1.4.0->-r requirement.txt (line 1)) (1.28.1)\n",
|
|
"Requirement already satisfied: termcolor in ./.local/lib/python3.8/site-packages (from fire>=0.3.1->kfp==1.4.0->-r requirement.txt (line 1)) (1.1.0)\n",
|
|
"Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from fire>=0.3.1->kfp==1.4.0->-r requirement.txt (line 1)) (1.15.0)\n",
|
|
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.8/site-packages (from google-auth>=1.6.1->kfp==1.4.0->-r requirement.txt (line 1)) (0.2.8)\n",
|
|
"Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.8/site-packages (from google-auth>=1.6.1->kfp==1.4.0->-r requirement.txt (line 1)) (4.7.2)\n",
|
|
"Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from google-auth>=1.6.1->kfp==1.4.0->-r requirement.txt (line 1)) (4.2.1)\n",
|
|
"Requirement already satisfied: setuptools>=40.3.0 in /opt/conda/lib/python3.8/site-packages (from google-auth>=1.6.1->kfp==1.4.0->-r requirement.txt (line 1)) (49.6.0.post20210108)\n",
|
|
"Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in ./.local/lib/python3.8/site-packages (from google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (2.26.0)\n",
|
|
"Requirement already satisfied: google-cloud-core<2.0dev,>=1.4.1 in /opt/conda/lib/python3.8/site-packages (from google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (1.6.0)\n",
|
|
"Requirement already satisfied: google-resumable-media<2.0dev,>=1.2.0 in /opt/conda/lib/python3.8/site-packages (from google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (1.2.0)\n",
|
|
"Requirement already satisfied: google-api-core<2.0.0dev,>=1.21.0 in /opt/conda/lib/python3.8/site-packages (from google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (1.26.3)\n",
|
|
"Requirement already satisfied: packaging>=14.3 in /opt/conda/lib/python3.8/site-packages (from google-api-core<2.0.0dev,>=1.21.0->google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (20.9)\n",
|
|
"Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /opt/conda/lib/python3.8/site-packages (from google-api-core<2.0.0dev,>=1.21.0->google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (1.53.0)\n",
|
|
"Requirement already satisfied: pytz in /opt/conda/lib/python3.8/site-packages (from google-api-core<2.0.0dev,>=1.21.0->google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (2021.1)\n",
|
|
"Requirement already satisfied: protobuf>=3.12.0 in /opt/conda/lib/python3.8/site-packages (from google-api-core<2.0.0dev,>=1.21.0->google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (3.15.7)\n",
|
|
"Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /opt/conda/lib/python3.8/site-packages (from google-resumable-media<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (1.1.2)\n",
|
|
"Requirement already satisfied: cffi>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from google-crc32c<2.0dev,>=1.0->google-resumable-media<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (1.14.5)\n",
|
|
"Requirement already satisfied: pycparser in /opt/conda/lib/python3.8/site-packages (from cffi>=1.0.0->google-crc32c<2.0dev,>=1.0->google-resumable-media<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (2.20)\n",
|
|
"Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema>=3.0.1->kfp==1.4.0->-r requirement.txt (line 1)) (0.17.3)\n",
|
|
"Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema>=3.0.1->kfp==1.4.0->-r requirement.txt (line 1)) (20.3.0)\n",
|
|
"Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.8/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.4.0->-r requirement.txt (line 1)) (2.8.1)\n",
|
|
"Requirement already satisfied: urllib3>=1.15 in /opt/conda/lib/python3.8/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.4.0->-r requirement.txt (line 1)) (1.26.4)\n",
|
|
"Requirement already satisfied: certifi in /opt/conda/lib/python3.8/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.4.0->-r requirement.txt (line 1)) (2020.12.5)\n",
|
|
"Requirement already satisfied: requests-oauthlib in /opt/conda/lib/python3.8/site-packages (from kubernetes<12.0.0,>=8.0.0->kfp==1.4.0->-r requirement.txt (line 1)) (1.3.0)\n",
|
|
"Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.8/site-packages (from kubernetes<12.0.0,>=8.0.0->kfp==1.4.0->-r requirement.txt (line 1)) (0.58.0)\n",
|
|
"Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging>=14.3->google-api-core<2.0.0dev,>=1.21.0->google-cloud-core<2.0dev,>=1.4.1->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (2.4.7)\n",
|
|
"Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.6.1->kfp==1.4.0->-r requirement.txt (line 1)) (0.4.8)\n",
|
|
"Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (2.10)\n",
|
|
"Requirement already satisfied: charset-normalizer~=2.0.0 in ./.local/lib/python3.8/site-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage>=1.13.0->kfp==1.4.0->-r requirement.txt (line 1)) (2.0.4)\n",
|
|
"Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.8/site-packages (from sklearn->-r requirement.txt (line 6)) (0.24.1)\n",
|
|
"Requirement already satisfied: wrapt<2,>=1.10 in /opt/conda/lib/python3.8/site-packages (from Deprecated->kfp==1.4.0->-r requirement.txt (line 1)) (1.12.1)\n",
|
|
"Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.8/site-packages (from requests-oauthlib->kubernetes<12.0.0,>=8.0.0->kfp==1.4.0->-r requirement.txt (line 1)) (3.1.0)\n",
|
|
"Requirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.8/site-packages (from scikit-learn->sklearn->-r requirement.txt (line 6)) (1.6.2)\n",
|
|
"Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.8/site-packages (from scikit-learn->sklearn->-r requirement.txt (line 6)) (1.0.1)\n",
|
|
"Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from scikit-learn->sklearn->-r requirement.txt (line 6)) (2.1.0)\n",
|
|
"Requirement already satisfied: wheel in /opt/conda/lib/python3.8/site-packages (from strip-hints->kfp==1.4.0->-r requirement.txt (line 1)) (0.36.2)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"with open(\"requirement.txt\", \"w\") as f:\n",
|
|
" f.write(\"kfp==1.4.0\\n\")\n",
|
|
" f.write(\"numpy\\n\")\n",
|
|
" f.write(\"keras\\n\")\n",
|
|
" f.write(\"tqdm\\n\")\n",
|
|
" f.write(\"config\\n\")\n",
|
|
" f.write(\"sklearn\\n\")\n",
|
|
"!pip install -r requirement.txt --upgrade --user\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 31,
|
|
"id": "d84bac6b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from typing import NamedTuple\n",
|
|
"import numpy\n",
|
|
"def load_data(log_folder:str)->NamedTuple('Outputs', [('start_time_string',str)]):\n",
|
|
" import numpy as np\n",
|
|
" import time\n",
|
|
" import sys\n",
|
|
" print(\"import done...\")\n",
|
|
" start = time.time()\n",
|
|
" data= np.load(\"triplet-data.npz\")\n",
|
|
" sys.path.append(\"./\")\n",
|
|
" \n",
|
|
"# from config import img_size, channel, faces_data_dir, FREEZE_LAYERS, classify, facenet_weight_path \n",
|
|
"# from inception_resnet_v1 import InceptionResNetV1\n",
|
|
"# from utils import scatter \n",
|
|
" X_train, X_test = data['arr_0'], data['arr_1']\n",
|
|
" print(X_train.shape, X_test.shape)\n",
|
|
" print(\"Saving data...\")\n",
|
|
" #print(X_train)\n",
|
|
" #print(X_test)\n",
|
|
" np.savez_compressed('/persist-log/triplet-data.npz', X_train, X_test)\n",
|
|
" print('Save complete ...')\n",
|
|
" start_time_string=str(start) #type is string\n",
|
|
" return [start_time_string]\n",
|
|
"\n",
|
|
"\n",
|
|
"def distributed_training_worker1(start_time_string:str)->NamedTuple('Outputs',[('model_path',str)]):\n",
|
|
" import numpy as np\n",
|
|
" import sys\n",
|
|
" import time\n",
|
|
" import tensorflow as tf\n",
|
|
" import json\n",
|
|
" import os\n",
|
|
" sys.path.append(\"./\")\n",
|
|
" sys.path.append(\"/persist-log\")\n",
|
|
" from config import img_size, channel, faces_data_dir, FREEZE_LAYERS, classify, facenet_weight_path\n",
|
|
" from inception_resnet_v1 import InceptionResNetV1\n",
|
|
" from itertools import permutations\n",
|
|
" from tqdm import tqdm\n",
|
|
" from tensorflow.keras import backend as K\n",
|
|
" from sklearn.manifold import TSNE\n",
|
|
" \n",
|
|
" #load data from pvc in the container\n",
|
|
" data = np.load('/persist-log/triplet-data.npz')\n",
|
|
" X_train, X_test = data['arr_0'], data['arr_1']\n",
|
|
"\n",
|
|
" def training_model(in_shape,freeze_layers,weights_path):\n",
|
|
"\n",
|
|
" def create_base_network(in_dims,freeze_layers,weights_path):\n",
|
|
" model = InceptionResNetV1(input_shape=in_dims, weights_path=weights_path)\n",
|
|
" print('layer length: ', len(model.layers))\n",
|
|
" for layer in model.layers[:freeze_layers]:\n",
|
|
" layer.trainable = False\n",
|
|
" for layer in model.layers[freeze_layers:]:\n",
|
|
" layer.trainable = True\n",
|
|
" return model\n",
|
|
" \n",
|
|
" def triplet_loss(y_true,y_pred,alpha=0.4):\n",
|
|
" total_lenght = y_pred.shape.as_list()[-1]\n",
|
|
" anchor = y_pred[:, 0:int(total_lenght * 1 / 3)]\n",
|
|
" positive = y_pred[:, int(total_lenght * 1 / 3):int(total_lenght * 2 / 3)]\n",
|
|
" negative = y_pred[:, int(total_lenght * 2 / 3):int(total_lenght * 3 / 3)]\n",
|
|
" # distance between the anchor and the positive\n",
|
|
" pos_dist = K.sum(K.square(anchor - positive), axis=1)\n",
|
|
" # distance between the anchor and the negative\n",
|
|
" neg_dist = K.sum(K.square(anchor - negative), axis=1)\n",
|
|
" # compute loss\n",
|
|
" basic_loss = pos_dist - neg_dist + alpha\n",
|
|
" loss = K.maximum(basic_loss, 0.0)\n",
|
|
" return loss\n",
|
|
" # define triplet input layers\n",
|
|
" anchor_input = tf.keras.layers.Input(in_shape, name='anchor_input')\n",
|
|
" positive_input = tf.keras.layers.Input(in_shape, name='positive_input')\n",
|
|
" negative_input = tf.keras.layers.Input(in_shape, name='negative_input')\n",
|
|
" Shared_DNN = create_base_network(in_shape, freeze_layers, weights_path)\n",
|
|
" # Shared_DNN.summary()\n",
|
|
" # encoded inputs\n",
|
|
" encoded_anchor = Shared_DNN(anchor_input)\n",
|
|
" encoded_positive = Shared_DNN(positive_input)\n",
|
|
" encoded_negative = Shared_DNN(negative_input)\n",
|
|
" # output\n",
|
|
" merged_vector = tf.keras.layers.concatenate([encoded_anchor, encoded_positive, encoded_negative],axis=-1,name='merged_layer')\n",
|
|
" model = tf.keras.Model(inputs=[anchor_input, positive_input, negative_input], outputs=merged_vector)\n",
|
|
" model.compile(\n",
|
|
" optimizer=adam_optim,\n",
|
|
" loss=triplet_loss,\n",
|
|
" )\n",
|
|
" return model\n",
|
|
" \n",
|
|
" \n",
|
|
" os.environ['TF_CONFIG'] = json.dumps({'cluster': {'worker': [\"pipeline-worker-1:3000\",\"pipeline-worker-2:3000\",\"pipeline-worker-3:3000\"]},'task': {'type': 'worker', 'index': 0}})\n",
|
|
" #os.environ['TF_CONFIG'] = json.dumps({'cluster': {'worker': [\"pipeline-worker-1:3000\"]},'task': {'type': 'worker', 'index': 0}})\n",
|
|
"\n",
|
|
"\n",
|
|
" strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy(\n",
|
|
" tf.distribute.experimental.CollectiveCommunication.RING)\n",
|
|
" NUM_WORKERS = strategy.num_replicas_in_sync\n",
|
|
" print('=================\\r\\nWorkers: ' + str(NUM_WORKERS) + '\\r\\n=================\\r\\n')\n",
|
|
" learn_rate = 0.0001 + NUM_WORKERS * 0.00006\n",
|
|
" adam_optim = tf.keras.optimizers.Adam(lr=learn_rate)\n",
|
|
" batch_size = 32* NUM_WORKERS\n",
|
|
" model_path='/persist-log/weight_tfdl.h5'\n",
|
|
" print(model_path)\n",
|
|
" callbacks = [tf.keras.callbacks.ModelCheckpoint(model_path, save_weights_only=True, verbose=1)]\n",
|
|
" #X_train=np.array(X_train)\n",
|
|
" #print(type(X_train))\n",
|
|
" with strategy.scope():\n",
|
|
" Anchor = X_train[:, 0, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Positive = X_train[:, 1, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Negative = X_train[:, 2, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Y_dummy = np.empty(Anchor.shape[0])\n",
|
|
" model = training_model((img_size, img_size, channel), FREEZE_LAYERS, facenet_weight_path)\n",
|
|
" \n",
|
|
" model.fit(x=[Anchor, Positive, Negative],\n",
|
|
" y=Y_dummy,\n",
|
|
" # Anchor_test = X_test[:, 0, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Positive_test = X_test[:, 1, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Negative_test = X_test[:, 2, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Y_dummy = np.empty(Anchor.shape[0])\n",
|
|
" # Y_dummy2 = np.empty((Anchor_test.shape[0], 1))\n",
|
|
" # validation_data=([Anchor_test,Positive_test,Negative_test],Y_dummy2),\n",
|
|
" # validation_split=0.2,\n",
|
|
" batch_size=batch_size, # old setting: 32\n",
|
|
" # steps_per_epoch=(X_train.shape[0] // batch_size) + 1,\n",
|
|
" epochs=10,\n",
|
|
" callbacks=callbacks\n",
|
|
" ) \n",
|
|
" end = time.time()\n",
|
|
" start_time_float=float(start_time_string)\n",
|
|
" print('execution time = ', ((end - start_time_float)/60))\n",
|
|
" return [model_path]\n",
|
|
"\n",
|
|
"\n",
|
|
"\n",
|
|
"\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 32,
|
|
"id": "97bad8e7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def distributed_training_worker2(start_time_string:str)->NamedTuple('Outputs',[('model_path_work2',str)]):\n",
|
|
" import numpy as np\n",
|
|
" import sys\n",
|
|
" import time\n",
|
|
" import tensorflow as tf\n",
|
|
" import json\n",
|
|
" import os\n",
|
|
" sys.path.append(\"./\")\n",
|
|
" sys.path.append(\"/persist-log\")\n",
|
|
" from config import img_size, channel, faces_data_dir, FREEZE_LAYERS, classify, facenet_weight_path\n",
|
|
" from inception_resnet_v1 import InceptionResNetV1\n",
|
|
" from itertools import permutations\n",
|
|
" from tqdm import tqdm\n",
|
|
" from tensorflow.keras import backend as K\n",
|
|
" from sklearn.manifold import TSNE\n",
|
|
" \n",
|
|
" #load data from pvc in the container\n",
|
|
" data = np.load('/persist-log/triplet-data.npz')\n",
|
|
" X_train, X_test = data['arr_0'], data['arr_1']\n",
|
|
"\n",
|
|
" def training_model(in_shape,freeze_layers,weights_path):\n",
|
|
"\n",
|
|
" def create_base_network(in_dims,freeze_layers,weights_path):\n",
|
|
" model = InceptionResNetV1(input_shape=in_dims, weights_path=weights_path)\n",
|
|
" print('layer length: ', len(model.layers))\n",
|
|
" for layer in model.layers[:freeze_layers]:\n",
|
|
" layer.trainable = False\n",
|
|
" for layer in model.layers[freeze_layers:]:\n",
|
|
" layer.trainable = True\n",
|
|
" return model\n",
|
|
" \n",
|
|
" def triplet_loss(y_true,y_pred,alpha=0.4):\n",
|
|
" total_lenght = y_pred.shape.as_list()[-1]\n",
|
|
" anchor = y_pred[:, 0:int(total_lenght * 1 / 3)]\n",
|
|
" positive = y_pred[:, int(total_lenght * 1 / 3):int(total_lenght * 2 / 3)]\n",
|
|
" negative = y_pred[:, int(total_lenght * 2 / 3):int(total_lenght * 3 / 3)]\n",
|
|
" # distance between the anchor and the positive\n",
|
|
" pos_dist = K.sum(K.square(anchor - positive), axis=1)\n",
|
|
" # distance between the anchor and the negative\n",
|
|
" neg_dist = K.sum(K.square(anchor - negative), axis=1)\n",
|
|
" # compute loss\n",
|
|
" basic_loss = pos_dist - neg_dist + alpha\n",
|
|
" loss = K.maximum(basic_loss, 0.0)\n",
|
|
" return loss\n",
|
|
" # define triplet input layers\n",
|
|
" anchor_input = tf.keras.layers.Input(in_shape, name='anchor_input')\n",
|
|
" positive_input = tf.keras.layers.Input(in_shape, name='positive_input')\n",
|
|
" negative_input = tf.keras.layers.Input(in_shape, name='negative_input')\n",
|
|
" Shared_DNN = create_base_network(in_shape, freeze_layers, weights_path)\n",
|
|
" # Shared_DNN.summary()\n",
|
|
" # encoded inputs\n",
|
|
" encoded_anchor = Shared_DNN(anchor_input)\n",
|
|
" encoded_positive = Shared_DNN(positive_input)\n",
|
|
" encoded_negative = Shared_DNN(negative_input)\n",
|
|
" # output\n",
|
|
" merged_vector = tf.keras.layers.concatenate([encoded_anchor, encoded_positive, encoded_negative],axis=-1,name='merged_layer')\n",
|
|
" model = tf.keras.Model(inputs=[anchor_input, positive_input, negative_input], outputs=merged_vector)\n",
|
|
" model.compile(\n",
|
|
" optimizer=adam_optim,\n",
|
|
" loss=triplet_loss,\n",
|
|
" )\n",
|
|
" return model\n",
|
|
" \n",
|
|
" \n",
|
|
" os.environ['TF_CONFIG'] = json.dumps({'cluster': {'worker': [\"pipeline-worker-1:3000\",\"pipeline-worker-2:3000\",\"pipeline-worker-3:3000\"]},'task': {'type': 'worker', 'index': 1}})\n",
|
|
"\n",
|
|
"\n",
|
|
" strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy(\n",
|
|
" tf.distribute.experimental.CollectiveCommunication.RING)\n",
|
|
" NUM_WORKERS = strategy.num_replicas_in_sync\n",
|
|
" print('=================\\r\\nWorkers: ' + str(NUM_WORKERS) + '\\r\\n=================\\r\\n')\n",
|
|
" learn_rate = 0.0001 + NUM_WORKERS * 0.00006\n",
|
|
" adam_optim = tf.keras.optimizers.Adam(lr=learn_rate)\n",
|
|
" batch_size = 32* NUM_WORKERS\n",
|
|
" model_path_work2='/persist-log/weight_tfdl.h5'\n",
|
|
"\n",
|
|
" callbacks = [tf.keras.callbacks.ModelCheckpoint(model_path_work2, save_weights_only=True, verbose=1)]\n",
|
|
" #X_train=np.array(X_train)\n",
|
|
" #print(type(X_train))\n",
|
|
" with strategy.scope():\n",
|
|
" Anchor = X_train[:, 0, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Positive = X_train[:, 1, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Negative = X_train[:, 2, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Y_dummy = np.empty(Anchor.shape[0])\n",
|
|
" model = training_model((img_size, img_size, channel), FREEZE_LAYERS, facenet_weight_path)\n",
|
|
" \n",
|
|
" model.fit(x=[Anchor, Positive, Negative],\n",
|
|
" y=Y_dummy,\n",
|
|
" # Anchor_test = X_test[:, 0, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Positive_test = X_test[:, 1, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Negative_test = X_test[:, 2, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Y_dummy = np.empty(Anchor.shape[0])\n",
|
|
" # Y_dummy2 = np.empty((Anchor_test.shape[0], 1))\n",
|
|
" # validation_data=([Anchor_test,Positive_test,Negative_test],Y_dummy2),\n",
|
|
" # validation_split=0.2,\n",
|
|
" batch_size=batch_size, # old setting: 32\n",
|
|
" # steps_per_epoch=(X_train.shape[0] // batch_size) + 1,\n",
|
|
" epochs=10,\n",
|
|
" callbacks=callbacks\n",
|
|
" ) \n",
|
|
" end = time.time()\n",
|
|
" start_time_float=float(start_time_string)\n",
|
|
" print('execution time = ', ((end - start_time_float)/60))\n",
|
|
" return [model_path_work2]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 33,
|
|
"id": "cd6214a6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def distributed_training_worker3(start_time_string:str)->NamedTuple('Outputs',[('model_path_work3',str)]):\n",
|
|
" import numpy as np\n",
|
|
" import sys\n",
|
|
" import time\n",
|
|
" import tensorflow as tf\n",
|
|
" import json\n",
|
|
" import os\n",
|
|
" sys.path.append(\"./\")\n",
|
|
" sys.path.append(\"/persist-log\")\n",
|
|
" from config import img_size, channel, faces_data_dir, FREEZE_LAYERS, classify, facenet_weight_path\n",
|
|
" from inception_resnet_v1 import InceptionResNetV1\n",
|
|
" from itertools import permutations\n",
|
|
" from tqdm import tqdm\n",
|
|
" from tensorflow.keras import backend as K\n",
|
|
" from sklearn.manifold import TSNE\n",
|
|
" \n",
|
|
" #load data from pvc in the container\n",
|
|
" data = np.load('/persist-log/triplet-data.npz')\n",
|
|
" X_train, X_test = data['arr_0'], data['arr_1']\n",
|
|
"\n",
|
|
" def training_model(in_shape,freeze_layers,weights_path):\n",
|
|
"\n",
|
|
" def create_base_network(in_dims,freeze_layers,weights_path):\n",
|
|
" model = InceptionResNetV1(input_shape=in_dims, weights_path=weights_path)\n",
|
|
" print('layer length: ', len(model.layers))\n",
|
|
" for layer in model.layers[:freeze_layers]:\n",
|
|
" layer.trainable = False\n",
|
|
" for layer in model.layers[freeze_layers:]:\n",
|
|
" layer.trainable = True\n",
|
|
" return model\n",
|
|
" \n",
|
|
" def triplet_loss(y_true,y_pred,alpha=0.4):\n",
|
|
" total_lenght = y_pred.shape.as_list()[-1]\n",
|
|
" anchor = y_pred[:, 0:int(total_lenght * 1 / 3)]\n",
|
|
" positive = y_pred[:, int(total_lenght * 1 / 3):int(total_lenght * 2 / 3)]\n",
|
|
" negative = y_pred[:, int(total_lenght * 2 / 3):int(total_lenght * 3 / 3)]\n",
|
|
" # distance between the anchor and the positive\n",
|
|
" pos_dist = K.sum(K.square(anchor - positive), axis=1)\n",
|
|
" # distance between the anchor and the negative\n",
|
|
" neg_dist = K.sum(K.square(anchor - negative), axis=1)\n",
|
|
" # compute loss\n",
|
|
" basic_loss = pos_dist - neg_dist + alpha\n",
|
|
" loss = K.maximum(basic_loss, 0.0)\n",
|
|
" return loss\n",
|
|
" # define triplet input layers\n",
|
|
" anchor_input = tf.keras.layers.Input(in_shape, name='anchor_input')\n",
|
|
" positive_input = tf.keras.layers.Input(in_shape, name='positive_input')\n",
|
|
" negative_input = tf.keras.layers.Input(in_shape, name='negative_input')\n",
|
|
" Shared_DNN = create_base_network(in_shape, freeze_layers, weights_path)\n",
|
|
" # Shared_DNN.summary()\n",
|
|
" # encoded inputs\n",
|
|
" encoded_anchor = Shared_DNN(anchor_input)\n",
|
|
" encoded_positive = Shared_DNN(positive_input)\n",
|
|
" encoded_negative = Shared_DNN(negative_input)\n",
|
|
" # output\n",
|
|
" merged_vector = tf.keras.layers.concatenate([encoded_anchor, encoded_positive, encoded_negative],axis=-1,name='merged_layer')\n",
|
|
" model = tf.keras.Model(inputs=[anchor_input, positive_input, negative_input], outputs=merged_vector)\n",
|
|
" model.compile(\n",
|
|
" optimizer=adam_optim,\n",
|
|
" loss=triplet_loss,\n",
|
|
" )\n",
|
|
" return model\n",
|
|
" \n",
|
|
" \n",
|
|
" os.environ['TF_CONFIG'] = json.dumps({'cluster': {'worker': [\"pipeline-worker-1:3000\",\"pipeline-worker-2:3000\",\"pipeline-worker-3:3000\"]},'task': {'type': 'worker', 'index': 2}})\n",
|
|
"\n",
|
|
"\n",
|
|
" strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy(\n",
|
|
" tf.distribute.experimental.CollectiveCommunication.RING)\n",
|
|
" NUM_WORKERS = strategy.num_replicas_in_sync\n",
|
|
" print('=================\\r\\nWorkers: ' + str(NUM_WORKERS) + '\\r\\n=================\\r\\n')\n",
|
|
" learn_rate = 0.0001 + NUM_WORKERS * 0.00006\n",
|
|
" adam_optim = tf.keras.optimizers.Adam(lr=learn_rate)\n",
|
|
" batch_size = 32* NUM_WORKERS\n",
|
|
" model_path_work3='/persist-log/weight_tfdl.h5'\n",
|
|
" callbacks = [tf.keras.callbacks.ModelCheckpoint(model_path_work3, save_weights_only=True, verbose=1)]\n",
|
|
" #X_train=np.array(X_train)\n",
|
|
" #print(type(X_train))\n",
|
|
" with strategy.scope():\n",
|
|
" Anchor = X_train[:, 0, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Positive = X_train[:, 1, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Negative = X_train[:, 2, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" Y_dummy = np.empty(Anchor.shape[0])\n",
|
|
" model = training_model((img_size, img_size, channel), FREEZE_LAYERS, facenet_weight_path)\n",
|
|
" \n",
|
|
" model.fit(x=[Anchor, Positive, Negative],\n",
|
|
" y=Y_dummy,\n",
|
|
" # Anchor_test = X_test[:, 0, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Positive_test = X_test[:, 1, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Negative_test = X_test[:, 2, :].reshape(-1, img_size, img_size, channel)\n",
|
|
" # Y_dummy = np.empty(Anchor.shape[0])\n",
|
|
" # Y_dummy2 = np.empty((Anchor_test.shape[0], 1))\n",
|
|
" # validation_data=([Anchor_test,Positive_test,Negative_test],Y_dummy2),\n",
|
|
" # validation_split=0.2,\n",
|
|
" batch_size=batch_size, # old setting: 32\n",
|
|
" # steps_per_epoch=(X_train.shape[0] // batch_size) + 1,\n",
|
|
" epochs=10,\n",
|
|
" callbacks=callbacks\n",
|
|
" ) \n",
|
|
" end = time.time()\n",
|
|
" start_time_float=float(start_time_string)\n",
|
|
" print('execution time = ', ((end - start_time_float)/60))\n",
|
|
" return [model_path_work3]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 34,
|
|
"id": "6a661a86",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def model_prediction(model_path:str,model_path_work2:str,model_path_work3:str)->NamedTuple('Outputs',[('model_path',str)]):\n",
|
|
" from os import listdir\n",
|
|
" from os.path import isfile\n",
|
|
" import time\n",
|
|
" import numpy as np\n",
|
|
" import cv2\n",
|
|
" from sklearn.manifold import TSNE\n",
|
|
" from scipy.spatial import distance\n",
|
|
" import tensorflow as tf\n",
|
|
" import sys\n",
|
|
" sys.path.append(\"./\")\n",
|
|
" sys.path.append(\"/persist-log\")\n",
|
|
" sys.path.append(\"/facenet/test\")\n",
|
|
" from img_process import align_image, prewhiten\n",
|
|
" from triplet_training import create_base_network\n",
|
|
" from utils import scatter\n",
|
|
" from config import img_size, channel, classify, FREEZE_LAYERS, facenet_weight_path, faces_data_dir\n",
|
|
" anchor_input = tf.keras.Input((img_size, img_size, channel,), name='anchor_input')\n",
|
|
" Shared_DNN = create_base_network((img_size, img_size, channel), FREEZE_LAYERS, facenet_weight_path)\n",
|
|
" encoded_anchor = Shared_DNN(anchor_input)\n",
|
|
" \n",
|
|
" model = tf.keras.Model(inputs=anchor_input, outputs=encoded_anchor)\n",
|
|
" model.load_weights(model_path)\n",
|
|
" model.summary()\n",
|
|
" start = time.time()\n",
|
|
" def l2_normalize(x, axis=-1, epsilon=1e-10):\n",
|
|
" output = x / np.sqrt(np.maximum(np.sum(np.square(x), axis=axis, keepdims=True), epsilon))\n",
|
|
" return output\n",
|
|
"\n",
|
|
"\n",
|
|
" # Acquire embedding from image\n",
|
|
" def embedding_extractor(img_path):\n",
|
|
" img = cv2.imread(img_path)\n",
|
|
" aligned = align_image(img)\n",
|
|
" #cv2.imwrite(\"facenet/align/\"+\"_aligned.jpg\", aligned)\n",
|
|
" if aligned is not None:\n",
|
|
" aligned = aligned.reshape(-1, img_size, img_size, channel)\n",
|
|
"\n",
|
|
" embs = l2_normalize(np.concatenate(model.predict(aligned)))\n",
|
|
" return embs\n",
|
|
" else:\n",
|
|
" print(img_path + ' is None')\n",
|
|
" return None\n",
|
|
" \n",
|
|
" testset_dir = 'facenet/test/'\n",
|
|
" items = listdir(testset_dir)\n",
|
|
"\n",
|
|
" jpgsList = [x for x in items if isfile(testset_dir + x)]\n",
|
|
" foldersList = [x for x in items if not isfile(testset_dir + x)]\n",
|
|
"\n",
|
|
" print(jpgsList)\n",
|
|
" print(foldersList)\n",
|
|
"\n",
|
|
" acc_total = 0\n",
|
|
" for i, anch_jpg in enumerate(jpgsList):\n",
|
|
" anchor_path = testset_dir + anch_jpg\n",
|
|
" anch_emb = embedding_extractor(anchor_path)\n",
|
|
"\n",
|
|
" for j, clt_folder in enumerate(foldersList):\n",
|
|
" clt_path = testset_dir + clt_folder + '/'\n",
|
|
" clt_jpgs = listdir(clt_path)\n",
|
|
" #print('anchor_path is :',anchor_path)\n",
|
|
" #print('clt_jpgs is :',clt_jpgs)\n",
|
|
" #print('clt_path is :',clt_path)\n",
|
|
"\n",
|
|
" str = anch_jpg\n",
|
|
" computeType = 1 if clt_folder == str.replace('.jpg', '') else 0\n",
|
|
"\n",
|
|
" loss = 0\n",
|
|
" if computeType == 1:\n",
|
|
" sum1 = 0\n",
|
|
" print('==============' + clt_folder + '&' + anch_jpg + '==============')\n",
|
|
" for k, clt_jpg in enumerate(clt_jpgs):\n",
|
|
" clt_jpg_path = clt_path + clt_jpg\n",
|
|
" clt_emb = embedding_extractor(clt_jpg_path)\n",
|
|
" distanceDiff = distance.euclidean(anch_emb, clt_emb) # calculate the distance\n",
|
|
" #print('distance = ', distanceDiff)\n",
|
|
" sum1 = distanceDiff + sum1\n",
|
|
" loss = loss + 1 if distanceDiff >= 1 else loss\n",
|
|
"\n",
|
|
" print(\"sum1\", sum1 / 50.0)\n",
|
|
" print('loss: ', loss)\n",
|
|
" accuracy = (len(clt_jpgs) - loss) / len(clt_jpgs)\n",
|
|
" print('accuracy: ', accuracy)\n",
|
|
" acc_total += accuracy\n",
|
|
" else:\n",
|
|
" print('==============' + clt_folder + '&' + anch_jpg + '==============')\n",
|
|
" sum2 = 0\n",
|
|
" for k, clt_jpg in enumerate(clt_jpgs):\n",
|
|
" clt_jpg_path = clt_path + clt_jpg\n",
|
|
" clt_emb = embedding_extractor(clt_jpg_path)\n",
|
|
" distanceDiff = distance.euclidean(anch_emb, clt_emb) # calculate the distance\n",
|
|
" #print('distance = ', distanceDiff)\n",
|
|
" loss = loss + 1 if distanceDiff < 1 else loss\n",
|
|
" sum2 = distanceDiff + sum2\n",
|
|
" print(\"sum2\", sum2 / 50.0)\n",
|
|
" print('loss: ', loss)\n",
|
|
" accuracy = (len(clt_jpgs) - loss) / len(clt_jpgs)\n",
|
|
" print('accuracy: ', accuracy)\n",
|
|
" acc_total += accuracy\n",
|
|
"\n",
|
|
" print('--acc_total', acc_total)\n",
|
|
"\n",
|
|
" acc_mean = acc_total / 81 * 100\n",
|
|
" print('final acc++------: ', acc_mean)\n",
|
|
" end = time.time()\n",
|
|
" print ('execution time', (end - start))\n",
|
|
" \n",
|
|
" return [model_path]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 35,
|
|
"id": "87cae256",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#serving\n",
|
|
"def serving(model_path:str, log_folder:str):\n",
|
|
" from flask import Flask,render_template,url_for,request,redirect,make_response,jsonify\n",
|
|
" from werkzeug.utils import secure_filename\n",
|
|
" import os \n",
|
|
" import cv2\n",
|
|
" import sys\n",
|
|
" import time\n",
|
|
" import base64\n",
|
|
" import math\n",
|
|
" from datetime import timedelta\n",
|
|
" import numpy as np\n",
|
|
" from os import listdir\n",
|
|
" from os.path import isfile\n",
|
|
" from sklearn.manifold import TSNE\n",
|
|
" from scipy.spatial import distance\n",
|
|
" import tensorflow as tf\n",
|
|
" \n",
|
|
" sys.path.append(\"./\")\n",
|
|
" sys.path.append(\"/persist-log\")\n",
|
|
" sys.path.append(\"/templates\")\n",
|
|
" \n",
|
|
" from img_process import align_image, prewhiten\n",
|
|
" from triplet_training import create_base_network\n",
|
|
" from utils import scatter\n",
|
|
" from config import img_size, channel, classify, FREEZE_LAYERS, facenet_weight_path, faces_data_dir\n",
|
|
" serving_time = time.time\n",
|
|
" ALLOWED_EXTENSIONS = set(['jpg','JPG'])\n",
|
|
" \n",
|
|
"\n",
|
|
" def allowed_file(filename):\n",
|
|
" return '.' in filename and filename.rsplit('.',1)[1] in ALLOWED_EXTENSIONS\n",
|
|
"\n",
|
|
"\n",
|
|
" def return_img_stream(img_local_path):\n",
|
|
" img_stream = ''\n",
|
|
" with open(img_local_path,'rb') as img_f:\n",
|
|
" img_stream = img_f.read()\n",
|
|
" img_stream = base64.b64encode(img_stream).decode()\n",
|
|
" return img_stream\n",
|
|
"\n",
|
|
" # L2 normalization\n",
|
|
" def l2_normalize(x, axis=-1, epsilon=1e-10):\n",
|
|
" output = x / np.sqrt(np.maximum(np.sum(np.square(x), axis=axis, keepdims=True), epsilon))\n",
|
|
" return output\n",
|
|
" \n",
|
|
"#--------------------------------------------------------------demo.py \n",
|
|
"\n",
|
|
" # Acquire embedding from image\n",
|
|
" def embedding_extractor(img_path,model):\n",
|
|
" img = cv2.imread(img_path)\n",
|
|
" aligned = align_image(img)\n",
|
|
" #cv2.imwrite(\"facenet/align/\"+\"_aligned.jpg\", aligned)\n",
|
|
" if aligned is not None:\n",
|
|
" aligned = aligned.reshape(-1, img_size, img_size, channel)\n",
|
|
"\n",
|
|
" embs = l2_normalize(np.concatenate(model.predict(aligned)))\n",
|
|
" return embs\n",
|
|
" else:\n",
|
|
" print(img_path + ' is None')\n",
|
|
" return None\n",
|
|
"#-------------------------------------------------------------flask\n",
|
|
"\n",
|
|
" app = Flask(__name__, template_folder=\"/templates\")\n",
|
|
"\n",
|
|
" app.send_file_max_age_default = timedelta(seconds=1)\n",
|
|
" \n",
|
|
" @app.route('/upload',methods=['GET','POST'])\n",
|
|
"\n",
|
|
" def upload():\n",
|
|
" img_stream = ''\n",
|
|
" loss = 0\n",
|
|
" distanceDiffbig = 0\n",
|
|
" distanceDiffsmall = 0\n",
|
|
" distance_sum = 0\n",
|
|
"\n",
|
|
" face = ''\n",
|
|
" face2 = ''\n",
|
|
" face3 = ''\n",
|
|
" acc_mean = 0\n",
|
|
"\n",
|
|
" distance_low1 = 0\n",
|
|
" distance_low2 = 0\n",
|
|
" distance_low3 = 0\n",
|
|
" distance_show1 = 2\n",
|
|
" distance_show2 = 2\n",
|
|
" distance_show3 = 2\n",
|
|
" \n",
|
|
" if request.method =='POST':\n",
|
|
" f = request.files['file']\n",
|
|
" user_input = request.form.get('name')\n",
|
|
" basepath = os.path.dirname(__file__)\n",
|
|
" sys.path.append('/facenet/test')\n",
|
|
" upload_path = os.path.join(basepath,'/facenet/test',secure_filename(f.filename))\n",
|
|
" print(basepath)\n",
|
|
" f.save(upload_path)\n",
|
|
" #start = time.time()\n",
|
|
" #model_path = '/persist-log/weight_tfdl.h5'\n",
|
|
" anchor_input = tf.keras.Input((img_size, img_size, channel,), name='anchor_input')\n",
|
|
" Shared_DNN = create_base_network((img_size, img_size, channel), FREEZE_LAYERS, facenet_weight_path)\n",
|
|
" encoded_anchor = Shared_DNN(anchor_input)\n",
|
|
"\n",
|
|
" model = tf.keras.Model(inputs=anchor_input, outputs=encoded_anchor)\n",
|
|
" model.load_weights(model_path) #/persist-log\n",
|
|
" model.summary()\n",
|
|
"\n",
|
|
" testset_dir = 'facenet/test/'\n",
|
|
" items = listdir(testset_dir)\n",
|
|
"\n",
|
|
" jpgsList = [x for x in items if isfile(testset_dir + x)]\n",
|
|
" foldersList = [x for x in items if not isfile(testset_dir + x)]\n",
|
|
"\n",
|
|
" print(jpgsList)\n",
|
|
" print(foldersList)\n",
|
|
" \n",
|
|
" acc_total = 0\n",
|
|
" img_stream = return_img_stream(upload_path)\n",
|
|
" for i, anch_jpg in enumerate(jpgsList):\n",
|
|
" #anchor_path = testset_dir + anch_jpg\n",
|
|
" anch_emb = embedding_extractor(upload_path,model)\n",
|
|
" \n",
|
|
" for j, clt_folder in enumerate(foldersList):\n",
|
|
" clt_path = testset_dir + clt_folder + '/'\n",
|
|
" clt_jpgs = listdir(clt_path)\n",
|
|
" str = anch_jpg\n",
|
|
" print('==============' + clt_folder + '&' + anch_jpg + '==============')\n",
|
|
" \n",
|
|
" for k, clt_jpg in enumerate(clt_jpgs):\n",
|
|
" clt_jpg_path = clt_path + clt_jpg\n",
|
|
" clt_emb = embedding_extractor(clt_jpg_path,model)\n",
|
|
" distanceDiff = distance.euclidean(anch_emb, clt_emb) # calculate the distance\n",
|
|
" distance_sum=distance_sum + distanceDiff\n",
|
|
" \n",
|
|
" if distanceDiff >= 1:\n",
|
|
" distanceDiffbig = distanceDiffbig + 1\n",
|
|
" \n",
|
|
" else:\n",
|
|
" distanceDiffsmall = distanceDiffsmall + 1\n",
|
|
" \n",
|
|
" if distanceDiffbig >= distanceDiffsmall :\n",
|
|
" loss = distanceDiffsmall\n",
|
|
" \n",
|
|
" else:\n",
|
|
" loss = distanceDiffbig\n",
|
|
" \n",
|
|
" distance_sum=distance_sum / 16 \n",
|
|
" \n",
|
|
" if distance_sum < distance_show3: \n",
|
|
" \n",
|
|
" if distance_sum < distance_show2:\n",
|
|
" \n",
|
|
" if distance_sum < distance_show1:\n",
|
|
" distance_show1 = distance_sum\n",
|
|
" distance_low1 = distance_sum\n",
|
|
" face = clt_folder\n",
|
|
" \n",
|
|
" else:\n",
|
|
" distance_low2 = distance_sum\n",
|
|
" distance_show2 = distance_sum\n",
|
|
" face2 = clt_folder\n",
|
|
" \n",
|
|
" else:\n",
|
|
" distance_show3 = distance_sum\n",
|
|
" distance_low3 = distance_sum\n",
|
|
" face3 = clt_folder\n",
|
|
" else:\n",
|
|
" distanceDiff = distanceDiff\n",
|
|
" \n",
|
|
" print('distance sum is:', distance_sum)\n",
|
|
" print('distanceDiffsmall = ', distanceDiffsmall)\n",
|
|
" print('distanceDiffbig = ', distanceDiffbig)\n",
|
|
" print( distanceDiff)\n",
|
|
" \n",
|
|
" distance_sum = 0\n",
|
|
" distanceDiffsmall = 0\n",
|
|
" distanceDiffbig = 0\n",
|
|
" print('loss: ', loss)\n",
|
|
" accuracy = (len(clt_jpgs) - loss) / len(clt_jpgs)\n",
|
|
" acc_total += accuracy\n",
|
|
" print('face = ', face)\n",
|
|
" print('The first is:',face,'distance is ',distance_low1)\n",
|
|
" print('The Second is:',face2,'distance is ',distance_low2)\n",
|
|
" print('The third is:',face3,'distance is ',distance_low3)\n",
|
|
" \n",
|
|
" distance_low1 = round(distance_low1,2) \n",
|
|
" distance_low2 = round(distance_low2,2)\n",
|
|
" distance_low3 = round(distance_low3,2)\n",
|
|
" acc_mean = acc_total / 9 * 100\n",
|
|
" acc_mean = round(acc_mean,2)\n",
|
|
" print('final acc++------: ', acc_mean)\n",
|
|
" os.remove(upload_path)\n",
|
|
" #end = time.time()\n",
|
|
" #print ('execution time', (end - serving_time))\n",
|
|
" \n",
|
|
" return render_template('upload.html',img_stream = img_stream, face = face , face2 = face2 , face3 = face3 , distance_low1 = distance_low1, distance_low2 = distance_low2 , distance_low3 = distance_low3, acc_mean = acc_mean )\n",
|
|
" \n",
|
|
" if __name__ == '__main__':\n",
|
|
" app.run(host = '127.0.0.1',port=8987,debug=True)\n",
|
|
" \n",
|
|
" return"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 36,
|
|
"id": "f8a12f54",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import kfp.dsl as dsl\n",
|
|
"import kfp.components as components\n",
|
|
"from typing import NamedTuple\n",
|
|
"import kfp\n",
|
|
"from kfp import dsl\n",
|
|
"from kfp.components import func_to_container_op, InputPath, OutputPath\n",
|
|
"from kubernetes.client.models import V1ContainerPort\n",
|
|
"@dsl.pipeline(\n",
|
|
" name='triplet_training pipeline',\n",
|
|
" description='triplet training test.'\n",
|
|
")\n",
|
|
"def triplet_training_pipeline():\n",
|
|
"\n",
|
|
" log_folder = '/persist-log'\n",
|
|
" pvc_name = \"triplet-trainaing-pvc\"\n",
|
|
" \n",
|
|
" #label name\n",
|
|
" name=\"pod-name\"\n",
|
|
" value1=\"worker-1\" # selector pod-name: worker-1\n",
|
|
" value2=\"worker-2\" # selector pod-name: worker-2\n",
|
|
" value3=\"worker-3\" # selector pod-name: worker-3\n",
|
|
" \n",
|
|
" container_port=3000\n",
|
|
" \n",
|
|
" #select node\n",
|
|
" label_name=\"disktype\"\n",
|
|
" label_value1=\"worker-1\" \n",
|
|
" label_value2=\"worker-2\" \n",
|
|
" label_value3=\"worker-3\" \n",
|
|
" \n",
|
|
" vop = dsl.VolumeOp(\n",
|
|
" name=pvc_name,\n",
|
|
" resource_name=\"newpvc\",\n",
|
|
" storage_class=\"managed-nfs-storage\",\n",
|
|
" size=\"30Gi\",\n",
|
|
" modes=dsl.VOLUME_MODE_RWM\n",
|
|
" )\n",
|
|
" \n",
|
|
" load_data_op=func_to_container_op(\n",
|
|
" func=load_data,\n",
|
|
" base_image=\"mike0355/k8s-facenet-distributed-training:4\", \n",
|
|
" )\n",
|
|
" \n",
|
|
" distributed_training_worker1_op=func_to_container_op(\n",
|
|
" func=distributed_training_worker1,\n",
|
|
" base_image=\"mike0355/k8s-facenet-distributed-training:4\"\n",
|
|
" )\n",
|
|
" \n",
|
|
" distributed_training_worker2_op=func_to_container_op(\n",
|
|
" func=distributed_training_worker2,\n",
|
|
" base_image=\"mike0355/k8s-facenet-distributed-training:4\"\n",
|
|
" )\n",
|
|
" \n",
|
|
" distributed_training_worker3_op=func_to_container_op(\n",
|
|
" func=distributed_training_worker3,\n",
|
|
" base_image=\"mike0355/k8s-facenet-distributed-training:4\"\n",
|
|
" ) \n",
|
|
" \n",
|
|
" model_prediction_op=func_to_container_op(\n",
|
|
" func=model_prediction,\n",
|
|
" base_image=\"mike0355/k8s-facenet-distributed-training:4\"\n",
|
|
" )\n",
|
|
" \n",
|
|
" serving_op=func_to_container_op(\n",
|
|
" func=serving,\n",
|
|
" base_image=\"mike0355/k8s-facenet-serving:3\"\n",
|
|
" )\n",
|
|
"\n",
|
|
" #----------------------------------------------------------task \n",
|
|
" load_data_task=load_data_op(log_folder).add_pvolumes({\n",
|
|
" log_folder:vop.volume,\n",
|
|
" })\n",
|
|
" \n",
|
|
" distributed_training_worker1_task=distributed_training_worker1_op(load_data_task.outputs['start_time_string']).add_pvolumes({ #woker1\n",
|
|
" log_folder:vop.volume,\n",
|
|
" }).add_pod_label(name,value1).add_node_selector_constraint(label_name,label_value1).add_port(V1ContainerPort(container_port=3000,host_port=3000))\n",
|
|
" \n",
|
|
" \n",
|
|
" distributed_training_worker2_task=distributed_training_worker2_op(load_data_task.outputs['start_time_string']).add_pvolumes({ #woker2\n",
|
|
" log_folder:vop.volume,\n",
|
|
" }).add_pod_label(name,value2).add_port(V1ContainerPort(container_port=3000,host_port=3000)).add_node_selector_constraint(label_name,label_value2)\n",
|
|
" \n",
|
|
" \n",
|
|
" distributed_training_worker3_task=distributed_training_worker3_op(load_data_task.outputs['start_time_string']).add_pvolumes({ #woker3\n",
|
|
" log_folder:vop.volume,\n",
|
|
" }).add_pod_label(name,value3).add_port(V1ContainerPort(container_port=3000,host_port=3000)).add_node_selector_constraint(label_name,label_value3)\n",
|
|
" \n",
|
|
" \n",
|
|
" model_prediction_task=model_prediction_op(distributed_training_worker1_task.outputs['model_path'],distributed_training_worker2_task.outputs['model_path_work2'], \n",
|
|
" distributed_training_worker3_task.outputs['model_path_work3']).add_pvolumes({\n",
|
|
" log_folder:vop.volume,\n",
|
|
" })\n",
|
|
" \n",
|
|
" \n",
|
|
" serving_task=serving_op(model_prediction_task.outputs['model_path'], log_folder).add_pvolumes({\n",
|
|
" log_folder:vop.volume,\n",
|
|
" })\n",
|
|
" \n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 37,
|
|
"id": "70db3347",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"kfp.compiler.Compiler().compile(triplet_training_pipeline, 'distributed-training-1011-final.yaml')\n",
|
|
"#kfp.compiler.Compiler().compile(triplet_training_pipeline, 'load-data0902.zip')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "071e9b42",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "860d92e2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"\n",
|
|
" \n",
|
|
" \n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b5df6fcc",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "283c28a6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.8.8"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|