Note

Go to the end to download the full example code

Segmentation tutorial

This example demonstrates image segmentation with an Akida-compatible model as illustrated through person segmentation using the Portrait128 dataset.

Using pre-trained models for quick runtime, this example shows the evolution of model performance for a trained keras floating point model, a keras quantized and Quantization Aware Trained (QAT) model, and an Akida-converted model. Notice that the performance of the original keras floating point model is maintained throughout the model conversion flow.

1. Load the dataset

import os
import numpy as np
from tensorflow.keras.utils import get_file

# Download validation set from Brainchip data server, it contains 10% of the original dataset
data_path = get_file("val.tar.gz",
                     "https://data.brainchip.com/dataset-mirror/portrait128/val.tar.gz",
                     cache_subdir=os.path.join("datasets", "portrait128"),
                     extract=True)

data_dir = os.path.join(os.path.dirname(data_path), "val")
x_val = np.load(os.path.join(data_dir, "val_img.npy"))
y_val = np.load(os.path.join(data_dir, "val_msk.npy")).astype('uint8')
batch_size = 32
steps = x_val.shape[0] // 32

# Visualize some data
import matplotlib.pyplot as plt

id = np.random.randint(0, x_val.shape[0])

fig, axs = plt.subplots(3, 3, constrained_layout=True)
for col in range(3):
    axs[0, col].imshow(x_val[id + col] / 255.)
    axs[0, col].axis('off')
    axs[1, col].imshow(1 - y_val[id + col], cmap='Greys')
    axs[1, col].axis('off')
    axs[2, col].imshow(x_val[id + col] / 255. * y_val[id + col])
    axs[2, col].axis('off')

fig.suptitle('Image, mask and masked image', fontsize=10)
plt.show()
Image, mask and masked image
Downloading data from https://data.brainchip.com/dataset-mirror/portrait128/val.tar.gz

     8192/267313385 [..............................] - ETA: 0s
    98304/267313385 [..............................] - ETA: 2:29
   516096/267313385 [..............................] - ETA: 54s 
  1048576/267313385 [..............................] - ETA: 39s
  1605632/267313385 [..............................] - ETA: 34s
  2154496/267313385 [..............................] - ETA: 31s
  2678784/267313385 [..............................] - ETA: 30s
  3194880/267313385 [..............................] - ETA: 29s
  3694592/267313385 [..............................] - ETA: 29s
  4218880/267313385 [..............................] - ETA: 28s
  4759552/267313385 [..............................] - ETA: 28s
  5267456/267313385 [..............................] - ETA: 27s
  5808128/267313385 [..............................] - ETA: 27s
  6365184/267313385 [..............................] - ETA: 27s
  6905856/267313385 [..............................] - ETA: 26s
  7462912/267313385 [..............................] - ETA: 26s
  8019968/267313385 [..............................] - ETA: 26s
  8577024/267313385 [..............................] - ETA: 26s
  9134080/267313385 [>.............................] - ETA: 25s
  9707520/267313385 [>.............................] - ETA: 25s
 10264576/267313385 [>.............................] - ETA: 25s
 10838016/267313385 [>.............................] - ETA: 25s
 11411456/267313385 [>.............................] - ETA: 25s
 11968512/267313385 [>.............................] - ETA: 24s
 12525568/267313385 [>.............................] - ETA: 24s
 13049856/267313385 [>.............................] - ETA: 24s
 13459456/267313385 [>.............................] - ETA: 25s
 14049280/267313385 [>.............................] - ETA: 25s
 14360576/267313385 [>.............................] - ETA: 25s
 14671872/267313385 [>.............................] - ETA: 26s
 14983168/267313385 [>.............................] - ETA: 26s
 15310848/267313385 [>.............................] - ETA: 26s
 15622144/267313385 [>.............................] - ETA: 26s
 15949824/267313385 [>.............................] - ETA: 27s
 16293888/267313385 [>.............................] - ETA: 27s
 16637952/267313385 [>.............................] - ETA: 27s
 16965632/267313385 [>.............................] - ETA: 27s
 17326080/267313385 [>.............................] - ETA: 27s
 17686528/267313385 [>.............................] - ETA: 27s
 18046976/267313385 [=>............................] - ETA: 27s
 18407424/267313385 [=>............................] - ETA: 28s
 18784256/267313385 [=>............................] - ETA: 28s
 19144704/267313385 [=>............................] - ETA: 28s
 19537920/267313385 [=>............................] - ETA: 28s
 19931136/267313385 [=>............................] - ETA: 28s
 20307968/267313385 [=>............................] - ETA: 28s
 20717568/267313385 [=>............................] - ETA: 28s
 21094400/267313385 [=>............................] - ETA: 28s
 21504000/267313385 [=>............................] - ETA: 28s
 21913600/267313385 [=>............................] - ETA: 28s
 22339584/267313385 [=>............................] - ETA: 28s
 22749184/267313385 [=>............................] - ETA: 28s
 23175168/267313385 [=>............................] - ETA: 28s
 23584768/267313385 [=>............................] - ETA: 28s
 24010752/267313385 [=>............................] - ETA: 28s
 24436736/267313385 [=>............................] - ETA: 28s
 24879104/267313385 [=>............................] - ETA: 28s
 25321472/267313385 [=>............................] - ETA: 28s
 25763840/267313385 [=>............................] - ETA: 28s
 26206208/267313385 [=>............................] - ETA: 28s
 26664960/267313385 [=>............................] - ETA: 27s
 27123712/267313385 [==>...........................] - ETA: 27s
 27598848/267313385 [==>...........................] - ETA: 27s
 28057600/267313385 [==>...........................] - ETA: 27s
 28549120/267313385 [==>...........................] - ETA: 27s
 29024256/267313385 [==>...........................] - ETA: 27s
 29515776/267313385 [==>...........................] - ETA: 27s
 30007296/267313385 [==>...........................] - ETA: 27s
 30498816/267313385 [==>...........................] - ETA: 27s
 30990336/267313385 [==>...........................] - ETA: 27s
 31481856/267313385 [==>...........................] - ETA: 27s
 31973376/267313385 [==>...........................] - ETA: 26s
 32481280/267313385 [==>...........................] - ETA: 26s
 32989184/267313385 [==>...........................] - ETA: 26s
 33497088/267313385 [==>...........................] - ETA: 26s
 34021376/267313385 [==>...........................] - ETA: 26s
 34529280/267313385 [==>...........................] - ETA: 26s
 35053568/267313385 [==>...........................] - ETA: 26s
 35594240/267313385 [==>...........................] - ETA: 26s
 36118528/267313385 [===>..........................] - ETA: 26s
 36659200/267313385 [===>..........................] - ETA: 25s
 37199872/267313385 [===>..........................] - ETA: 25s
 37756928/267313385 [===>..........................] - ETA: 25s
 38297600/267313385 [===>..........................] - ETA: 25s
 38862848/267313385 [===>..........................] - ETA: 25s
 39411712/267313385 [===>..........................] - ETA: 25s
 39952384/267313385 [===>..........................] - ETA: 25s
 40509440/267313385 [===>..........................] - ETA: 25s
 41050112/267313385 [===>..........................] - ETA: 24s
 41615360/267313385 [===>..........................] - ETA: 24s
 42164224/267313385 [===>..........................] - ETA: 24s
 42721280/267313385 [===>..........................] - ETA: 24s
 43311104/267313385 [===>..........................] - ETA: 24s
 43884544/267313385 [===>..........................] - ETA: 24s
 44441600/267313385 [===>..........................] - ETA: 24s
 45015040/267313385 [====>.........................] - ETA: 24s
 45572096/267313385 [====>.........................] - ETA: 23s
 46129152/267313385 [====>.........................] - ETA: 23s
 46686208/267313385 [====>.........................] - ETA: 23s
 47259648/267313385 [====>.........................] - ETA: 23s
 47833088/267313385 [====>.........................] - ETA: 23s
 48390144/267313385 [====>.........................] - ETA: 23s
 48955392/267313385 [====>.........................] - ETA: 23s
 49520640/267313385 [====>.........................] - ETA: 23s
 50094080/267313385 [====>.........................] - ETA: 23s
 50651136/267313385 [====>.........................] - ETA: 23s
 51208192/267313385 [====>.........................] - ETA: 22s
 51765248/267313385 [====>.........................] - ETA: 22s
 52338688/267313385 [====>.........................] - ETA: 22s
 52895744/267313385 [====>.........................] - ETA: 22s
 53256192/267313385 [====>.........................] - ETA: 22s
 53960704/267313385 [=====>........................] - ETA: 22s
 54435840/267313385 [=====>........................] - ETA: 22s
 54927360/267313385 [=====>........................] - ETA: 22s
 55418880/267313385 [=====>........................] - ETA: 22s
 55910400/267313385 [=====>........................] - ETA: 22s
 56418304/267313385 [=====>........................] - ETA: 22s
 56795136/267313385 [=====>........................] - ETA: 22s
 57270272/267313385 [=====>........................] - ETA: 22s
 57614336/267313385 [=====>........................] - ETA: 22s
 57942016/267313385 [=====>........................] - ETA: 22s
 58351616/267313385 [=====>........................] - ETA: 22s
 58695680/267313385 [=====>........................] - ETA: 22s
 59121664/267313385 [=====>........................] - ETA: 22s
 59465728/267313385 [=====>........................] - ETA: 22s
 59891712/267313385 [=====>........................] - ETA: 22s
 60252160/267313385 [=====>........................] - ETA: 22s
 60678144/267313385 [=====>........................] - ETA: 22s
 61038592/267313385 [=====>........................] - ETA: 22s
 61431808/267313385 [=====>........................] - ETA: 22s
 61857792/267313385 [=====>........................] - ETA: 22s
 62300160/267313385 [=====>........................] - ETA: 22s
 62693376/267313385 [======>.......................] - ETA: 22s
 63135744/267313385 [======>.......................] - ETA: 22s
 63561728/267313385 [======>.......................] - ETA: 22s
 64004096/267313385 [======>.......................] - ETA: 21s
 64446464/267313385 [======>.......................] - ETA: 21s
 64888832/267313385 [======>.......................] - ETA: 21s
 65347584/267313385 [======>.......................] - ETA: 21s
 65806336/267313385 [======>.......................] - ETA: 21s
 66265088/267313385 [======>.......................] - ETA: 21s
 66740224/267313385 [======>.......................] - ETA: 21s
 67198976/267313385 [======>.......................] - ETA: 21s
 67641344/267313385 [======>.......................] - ETA: 21s
 68132864/267313385 [======>.......................] - ETA: 21s
 68608000/267313385 [======>.......................] - ETA: 21s
 69083136/267313385 [======>.......................] - ETA: 21s
 69591040/267313385 [======>.......................] - ETA: 21s
 70066176/267313385 [======>.......................] - ETA: 21s
 70574080/267313385 [======>.......................] - ETA: 21s
 71081984/267313385 [======>.......................] - ETA: 21s
 71589888/267313385 [=======>......................] - ETA: 21s
 72097792/267313385 [=======>......................] - ETA: 21s
 72605696/267313385 [=======>......................] - ETA: 21s
 73113600/267313385 [=======>......................] - ETA: 20s
 73605120/267313385 [=======>......................] - ETA: 20s
 74145792/267313385 [=======>......................] - ETA: 20s
 74670080/267313385 [=======>......................] - ETA: 20s
 75210752/267313385 [=======>......................] - ETA: 20s
 75751424/267313385 [=======>......................] - ETA: 20s
 76292096/267313385 [=======>......................] - ETA: 20s
 76849152/267313385 [=======>......................] - ETA: 20s
 77406208/267313385 [=======>......................] - ETA: 20s
 77946880/267313385 [=======>......................] - ETA: 20s
 78520320/267313385 [=======>......................] - ETA: 20s
 79077376/267313385 [=======>......................] - ETA: 20s
 79634432/267313385 [=======>......................] - ETA: 20s
 80158720/267313385 [=======>......................] - ETA: 19s
 80732160/267313385 [========>.....................] - ETA: 19s
 81289216/267313385 [========>.....................] - ETA: 19s
 81846272/267313385 [========>.....................] - ETA: 19s
 82419712/267313385 [========>.....................] - ETA: 19s
 82993152/267313385 [========>.....................] - ETA: 19s
 83566592/267313385 [========>.....................] - ETA: 19s
 84123648/267313385 [========>.....................] - ETA: 19s
 84680704/267313385 [========>.....................] - ETA: 19s
 85254144/267313385 [========>.....................] - ETA: 19s
 85811200/267313385 [========>.....................] - ETA: 19s
 86368256/267313385 [========>.....................] - ETA: 19s
 86941696/267313385 [========>.....................] - ETA: 19s
 87482368/267313385 [========>.....................] - ETA: 18s
 88055808/267313385 [========>.....................] - ETA: 18s
 88612864/267313385 [========>.....................] - ETA: 18s
 89169920/267313385 [=========>....................] - ETA: 18s
 89726976/267313385 [=========>....................] - ETA: 18s
 90300416/267313385 [=========>....................] - ETA: 18s
 90857472/267313385 [=========>....................] - ETA: 18s
 91299840/267313385 [=========>....................] - ETA: 18s
 92020736/267313385 [=========>....................] - ETA: 18s
 92479488/267313385 [=========>....................] - ETA: 18s
 92971008/267313385 [=========>....................] - ETA: 18s
 93446144/267313385 [=========>....................] - ETA: 18s
 93937664/267313385 [=========>....................] - ETA: 18s
 94445568/267313385 [=========>....................] - ETA: 18s
 94953472/267313385 [=========>....................] - ETA: 18s
 95444992/267313385 [=========>....................] - ETA: 17s
 95936512/267313385 [=========>....................] - ETA: 17s
 96444416/267313385 [=========>....................] - ETA: 17s
 96968704/267313385 [=========>....................] - ETA: 17s
 97476608/267313385 [=========>....................] - ETA: 17s
 98017280/267313385 [==========>...................] - ETA: 17s
 98541568/267313385 [==========>...................] - ETA: 17s
 99082240/267313385 [==========>...................] - ETA: 17s
 99491840/267313385 [==========>...................] - ETA: 17s
 99835904/267313385 [==========>...................] - ETA: 17s
100229120/267313385 [==========>...................] - ETA: 17s
100622336/267313385 [==========>...................] - ETA: 17s
101015552/267313385 [==========>...................] - ETA: 17s
101343232/267313385 [==========>...................] - ETA: 17s
101720064/267313385 [==========>...................] - ETA: 17s
101900288/267313385 [==========>...................] - ETA: 17s
102064128/267313385 [==========>...................] - ETA: 17s
102244352/267313385 [==========>...................] - ETA: 17s
102424576/267313385 [==========>...................] - ETA: 17s
102621184/267313385 [==========>...................] - ETA: 17s
102817792/267313385 [==========>...................] - ETA: 17s
103014400/267313385 [==========>...................] - ETA: 17s
103211008/267313385 [==========>...................] - ETA: 17s
103374848/267313385 [==========>...................] - ETA: 17s
103587840/267313385 [==========>...................] - ETA: 17s
103800832/267313385 [==========>...................] - ETA: 17s
104013824/267313385 [==========>...................] - ETA: 17s
104210432/267313385 [==========>...................] - ETA: 17s
104423424/267313385 [==========>...................] - ETA: 17s
104620032/267313385 [==========>...................] - ETA: 17s
104849408/267313385 [==========>...................] - ETA: 18s
105046016/267313385 [==========>...................] - ETA: 18s
105308160/267313385 [==========>...................] - ETA: 18s
105570304/267313385 [==========>...................] - ETA: 18s
105816064/267313385 [==========>...................] - ETA: 18s
106061824/267313385 [==========>...................] - ETA: 18s
106323968/267313385 [==========>...................] - ETA: 18s
106569728/267313385 [==========>...................] - ETA: 18s
106848256/267313385 [==========>...................] - ETA: 18s
107094016/267313385 [===========>..................] - ETA: 18s
107372544/267313385 [===========>..................] - ETA: 18s
107667456/267313385 [===========>..................] - ETA: 18s
107929600/267313385 [===========>..................] - ETA: 18s
108224512/267313385 [===========>..................] - ETA: 18s
108519424/267313385 [===========>..................] - ETA: 18s
108781568/267313385 [===========>..................] - ETA: 18s
109109248/267313385 [===========>..................] - ETA: 18s
109371392/267313385 [===========>..................] - ETA: 18s
109699072/267313385 [===========>..................] - ETA: 18s
109977600/267313385 [===========>..................] - ETA: 18s
110288896/267313385 [===========>..................] - ETA: 18s
110600192/267313385 [===========>..................] - ETA: 18s
110927872/267313385 [===========>..................] - ETA: 18s
111255552/267313385 [===========>..................] - ETA: 17s
111616000/267313385 [===========>..................] - ETA: 17s
111910912/267313385 [===========>..................] - ETA: 17s
112254976/267313385 [===========>..................] - ETA: 17s
112582656/267313385 [===========>..................] - ETA: 17s
112943104/267313385 [===========>..................] - ETA: 17s
113303552/267313385 [===========>..................] - ETA: 17s
113664000/267313385 [===========>..................] - ETA: 17s
114040832/267313385 [===========>..................] - ETA: 17s
114401280/267313385 [===========>..................] - ETA: 17s
114761728/267313385 [===========>..................] - ETA: 17s
115171328/267313385 [===========>..................] - ETA: 17s
115531776/267313385 [===========>..................] - ETA: 17s
115941376/267313385 [============>.................] - ETA: 17s
116301824/267313385 [============>.................] - ETA: 17s
116711424/267313385 [============>.................] - ETA: 17s
117088256/267313385 [============>.................] - ETA: 17s
117514240/267313385 [============>.................] - ETA: 17s
117907456/267313385 [============>.................] - ETA: 17s
118325248/267313385 [============>.................] - ETA: 17s
118718464/267313385 [============>.................] - ETA: 17s
119103488/267313385 [============>.................] - ETA: 17s
119545856/267313385 [============>.................] - ETA: 17s
119955456/267313385 [============>.................] - ETA: 17s
120414208/267313385 [============>.................] - ETA: 17s
120840192/267313385 [============>.................] - ETA: 17s
121282560/267313385 [============>.................] - ETA: 17s
121741312/267313385 [============>.................] - ETA: 16s
122216448/267313385 [============>.................] - ETA: 16s
122658816/267313385 [============>.................] - ETA: 16s
123133952/267313385 [============>.................] - ETA: 16s
123592704/267313385 [============>.................] - ETA: 16s
124076032/267313385 [============>.................] - ETA: 16s
124542976/267313385 [============>.................] - ETA: 16s
125018112/267313385 [=============>................] - ETA: 16s
125493248/267313385 [=============>................] - ETA: 16s
125984768/267313385 [=============>................] - ETA: 16s
126459904/267313385 [=============>................] - ETA: 16s
126951424/267313385 [=============>................] - ETA: 16s
127459328/267313385 [=============>................] - ETA: 16s
127950848/267313385 [=============>................] - ETA: 16s
128442368/267313385 [=============>................] - ETA: 16s
128950272/267313385 [=============>................] - ETA: 16s
129458176/267313385 [=============>................] - ETA: 16s
129966080/267313385 [=============>................] - ETA: 15s
130490368/267313385 [=============>................] - ETA: 15s
131014656/267313385 [=============>................] - ETA: 15s
131538944/267313385 [=============>................] - ETA: 15s
132063232/267313385 [=============>................] - ETA: 15s
132603904/267313385 [=============>................] - ETA: 15s
133144576/267313385 [=============>................] - ETA: 15s
133685248/267313385 [==============>...............] - ETA: 15s
134242304/267313385 [==============>...............] - ETA: 15s
134782976/267313385 [==============>...............] - ETA: 15s
135340032/267313385 [==============>...............] - ETA: 15s
135880704/267313385 [==============>...............] - ETA: 15s
136437760/267313385 [==============>...............] - ETA: 15s
136994816/267313385 [==============>...............] - ETA: 14s
137568256/267313385 [==============>...............] - ETA: 14s
138125312/267313385 [==============>...............] - ETA: 14s
138682368/267313385 [==============>...............] - ETA: 14s
139255808/267313385 [==============>...............] - ETA: 14s
139829248/267313385 [==============>...............] - ETA: 14s
140402688/267313385 [==============>...............] - ETA: 14s
140976128/267313385 [==============>...............] - ETA: 14s
141533184/267313385 [==============>...............] - ETA: 14s
142106624/267313385 [==============>...............] - ETA: 14s
142663680/267313385 [===============>..............] - ETA: 14s
143228928/267313385 [===============>..............] - ETA: 14s
143777792/267313385 [===============>..............] - ETA: 14s
144334848/267313385 [===============>..............] - ETA: 13s
144908288/267313385 [===============>..............] - ETA: 13s
145481728/267313385 [===============>..............] - ETA: 13s
146038784/267313385 [===============>..............] - ETA: 13s
146612224/267313385 [===============>..............] - ETA: 13s
147177472/267313385 [===============>..............] - ETA: 13s
147726336/267313385 [===============>..............] - ETA: 13s
148283392/267313385 [===============>..............] - ETA: 13s
148856832/267313385 [===============>..............] - ETA: 13s
149233664/267313385 [===============>..............] - ETA: 13s
149921792/267313385 [===============>..............] - ETA: 13s
150413312/267313385 [===============>..............] - ETA: 13s
150888448/267313385 [===============>..............] - ETA: 13s
151396352/267313385 [===============>..............] - ETA: 13s
151887872/267313385 [================>.............] - ETA: 13s
152395776/267313385 [================>.............] - ETA: 12s
152903680/267313385 [================>.............] - ETA: 12s
153427968/267313385 [================>.............] - ETA: 12s
153952256/267313385 [================>.............] - ETA: 12s
154460160/267313385 [================>.............] - ETA: 12s
154984448/267313385 [================>.............] - ETA: 12s
155525120/267313385 [================>.............] - ETA: 12s
156065792/267313385 [================>.............] - ETA: 12s
156606464/267313385 [================>.............] - ETA: 12s
157147136/267313385 [================>.............] - ETA: 12s
157704192/267313385 [================>.............] - ETA: 12s
158244864/267313385 [================>.............] - ETA: 12s
158785536/267313385 [================>.............] - ETA: 12s
159326208/267313385 [================>.............] - ETA: 12s
159883264/267313385 [================>.............] - ETA: 12s
160407552/267313385 [=================>............] - ETA: 11s
160964608/267313385 [=================>............] - ETA: 11s
161505280/267313385 [=================>............] - ETA: 11s
162078720/267313385 [=================>............] - ETA: 11s
162619392/267313385 [=================>............] - ETA: 11s
163176448/267313385 [=================>............] - ETA: 11s
163733504/267313385 [=================>............] - ETA: 11s
164274176/267313385 [=================>............] - ETA: 11s
164847616/267313385 [=================>............] - ETA: 11s
165421056/267313385 [=================>............] - ETA: 11s
165961728/267313385 [=================>............] - ETA: 11s
166535168/267313385 [=================>............] - ETA: 11s
167092224/267313385 [=================>............] - ETA: 11s
167649280/267313385 [=================>............] - ETA: 11s
168222720/267313385 [=================>............] - ETA: 10s
168779776/267313385 [=================>............] - ETA: 10s
169353216/267313385 [==================>...........] - ETA: 10s
169926656/267313385 [==================>...........] - ETA: 10s
170483712/267313385 [==================>...........] - ETA: 10s
171040768/267313385 [==================>...........] - ETA: 10s
171597824/267313385 [==================>...........] - ETA: 10s
172171264/267313385 [==================>...........] - ETA: 10s
172728320/267313385 [==================>...........] - ETA: 10s
173301760/267313385 [==================>...........] - ETA: 10s
173891584/267313385 [==================>...........] - ETA: 10s
174415872/267313385 [==================>...........] - ETA: 10s
174874624/267313385 [==================>...........] - ETA: 10s
175349760/267313385 [==================>...........] - ETA: 10s
175808512/267313385 [==================>...........] - ETA: 10s
176283648/267313385 [==================>...........] - ETA: 10s
176758784/267313385 [==================>...........] - ETA: 9s 
177250304/267313385 [==================>...........] - ETA: 9s
177725440/267313385 [==================>...........] - ETA: 9s
178233344/267313385 [===================>..........] - ETA: 9s
178708480/267313385 [===================>..........] - ETA: 9s
179216384/267313385 [===================>..........] - ETA: 9s
179724288/267313385 [===================>..........] - ETA: 9s
180248576/267313385 [===================>..........] - ETA: 9s
180772864/267313385 [===================>..........] - ETA: 9s
181297152/267313385 [===================>..........] - ETA: 9s
181805056/267313385 [===================>..........] - ETA: 9s
182345728/267313385 [===================>..........] - ETA: 9s
182870016/267313385 [===================>..........] - ETA: 9s
183410688/267313385 [===================>..........] - ETA: 9s
183934976/267313385 [===================>..........] - ETA: 9s
184492032/267313385 [===================>..........] - ETA: 9s
185032704/267313385 [===================>..........] - ETA: 9s
185573376/267313385 [===================>..........] - ETA: 8s
186114048/267313385 [===================>..........] - ETA: 8s
186671104/267313385 [===================>..........] - ETA: 8s
187244544/267313385 [====================>.........] - ETA: 8s
187801600/267313385 [====================>.........] - ETA: 8s
188358656/267313385 [====================>.........] - ETA: 8s
188932096/267313385 [====================>.........] - ETA: 8s
189489152/267313385 [====================>.........] - ETA: 8s
190046208/267313385 [====================>.........] - ETA: 8s
190619648/267313385 [====================>.........] - ETA: 8s
191160320/267313385 [====================>.........] - ETA: 8s
191733760/267313385 [====================>.........] - ETA: 8s
192290816/267313385 [====================>.........] - ETA: 8s
192856064/267313385 [====================>.........] - ETA: 8s
193421312/267313385 [====================>.........] - ETA: 8s
193994752/267313385 [====================>.........] - ETA: 7s
194551808/267313385 [====================>.........] - ETA: 7s
195125248/267313385 [====================>.........] - ETA: 7s
195682304/267313385 [====================>.........] - ETA: 7s
196222976/267313385 [=====================>........] - ETA: 7s
196714496/267313385 [=====================>........] - ETA: 7s
197173248/267313385 [=====================>........] - ETA: 7s
197632000/267313385 [=====================>........] - ETA: 7s
198107136/267313385 [=====================>........] - ETA: 7s
198582272/267313385 [=====================>........] - ETA: 7s
199057408/267313385 [=====================>........] - ETA: 7s
199548928/267313385 [=====================>........] - ETA: 7s
200024064/267313385 [=====================>........] - ETA: 7s
200515584/267313385 [=====================>........] - ETA: 7s
201007104/267313385 [=====================>........] - ETA: 7s
201482240/267313385 [=====================>........] - ETA: 7s
201973760/267313385 [=====================>........] - ETA: 7s
202481664/267313385 [=====================>........] - ETA: 7s
202989568/267313385 [=====================>........] - ETA: 6s
203513856/267313385 [=====================>........] - ETA: 6s
204038144/267313385 [=====================>........] - ETA: 6s
204562432/267313385 [=====================>........] - ETA: 6s
205086720/267313385 [======================>.......] - ETA: 6s
205627392/267313385 [======================>.......] - ETA: 6s
206151680/267313385 [======================>.......] - ETA: 6s
206708736/267313385 [======================>.......] - ETA: 6s
207249408/267313385 [======================>.......] - ETA: 6s
207806464/267313385 [======================>.......] - ETA: 6s
208330752/267313385 [======================>.......] - ETA: 6s
208855040/267313385 [======================>.......] - ETA: 6s
209362944/267313385 [======================>.......] - ETA: 6s
209903616/267313385 [======================>.......] - ETA: 6s
210231296/267313385 [======================>.......] - ETA: 6s
210821120/267313385 [======================>.......] - ETA: 6s
211247104/267313385 [======================>.......] - ETA: 6s
211656704/267313385 [======================>.......] - ETA: 6s
212082688/267313385 [======================>.......] - ETA: 5s
212508672/267313385 [======================>.......] - ETA: 5s
212934656/267313385 [======================>.......] - ETA: 5s
213360640/267313385 [======================>.......] - ETA: 5s
213803008/267313385 [======================>.......] - ETA: 5s
214245376/267313385 [=======================>......] - ETA: 5s
214687744/267313385 [=======================>......] - ETA: 5s
215146496/267313385 [=======================>......] - ETA: 5s
215572480/267313385 [=======================>......] - ETA: 5s
216047616/267313385 [=======================>......] - ETA: 5s
216506368/267313385 [=======================>......] - ETA: 5s
216981504/267313385 [=======================>......] - ETA: 5s
217456640/267313385 [=======================>......] - ETA: 5s
217931776/267313385 [=======================>......] - ETA: 5s
218423296/267313385 [=======================>......] - ETA: 5s
218898432/267313385 [=======================>......] - ETA: 5s
219389952/267313385 [=======================>......] - ETA: 5s
219865088/267313385 [=======================>......] - ETA: 5s
220372992/267313385 [=======================>......] - ETA: 5s
220815360/267313385 [=======================>......] - ETA: 5s
221356032/267313385 [=======================>......] - ETA: 4s
221700096/267313385 [=======================>......] - ETA: 4s
222060544/267313385 [=======================>......] - ETA: 4s
222437376/267313385 [=======================>......] - ETA: 4s
222781440/267313385 [========================>.....] - ETA: 4s
223182848/267313385 [========================>.....] - ETA: 4s
223518720/267313385 [========================>.....] - ETA: 4s
223928320/267313385 [========================>.....] - ETA: 4s
224288768/267313385 [========================>.....] - ETA: 4s
224681984/267313385 [========================>.....] - ETA: 4s
225075200/267313385 [========================>.....] - ETA: 4s
225484800/267313385 [========================>.....] - ETA: 4s
225878016/267313385 [========================>.....] - ETA: 4s
226304000/267313385 [========================>.....] - ETA: 4s
226697216/267313385 [========================>.....] - ETA: 4s
227123200/267313385 [========================>.....] - ETA: 4s
227549184/267313385 [========================>.....] - ETA: 4s
227975168/267313385 [========================>.....] - ETA: 4s
228417536/267313385 [========================>.....] - ETA: 4s
228827136/267313385 [========================>.....] - ETA: 4s
229269504/267313385 [========================>.....] - ETA: 4s
229695488/267313385 [========================>.....] - ETA: 4s
230137856/267313385 [========================>.....] - ETA: 4s
230596608/267313385 [========================>.....] - ETA: 3s
231055360/267313385 [========================>.....] - ETA: 3s
231497728/267313385 [========================>.....] - ETA: 3s
231972864/267313385 [=========================>....] - ETA: 3s
232448000/267313385 [=========================>....] - ETA: 3s
232923136/267313385 [=========================>....] - ETA: 3s
233316352/267313385 [=========================>....] - ETA: 3s
233857024/267313385 [=========================>....] - ETA: 3s
234233856/267313385 [=========================>....] - ETA: 3s
234561536/267313385 [=========================>....] - ETA: 3s
234954752/267313385 [=========================>....] - ETA: 3s
235298816/267313385 [=========================>....] - ETA: 3s
235692032/267313385 [=========================>....] - ETA: 3s
236068864/267313385 [=========================>....] - ETA: 3s
236445696/267313385 [=========================>....] - ETA: 3s
236822528/267313385 [=========================>....] - ETA: 3s
237215744/267313385 [=========================>....] - ETA: 3s
237608960/267313385 [=========================>....] - ETA: 3s
238002176/267313385 [=========================>....] - ETA: 3s
238395392/267313385 [=========================>....] - ETA: 3s
238821376/267313385 [=========================>....] - ETA: 3s
239214592/267313385 [=========================>....] - ETA: 3s
239656960/267313385 [=========================>....] - ETA: 3s
240091136/267313385 [=========================>....] - ETA: 2s
240508928/267313385 [=========================>....] - ETA: 2s
240951296/267313385 [==========================>...] - ETA: 2s
241377280/267313385 [==========================>...] - ETA: 2s
241819648/267313385 [==========================>...] - ETA: 2s
242245632/267313385 [==========================>...] - ETA: 2s
242704384/267313385 [==========================>...] - ETA: 2s
243146752/267313385 [==========================>...] - ETA: 2s
243605504/267313385 [==========================>...] - ETA: 2s
244064256/267313385 [==========================>...] - ETA: 2s
244523008/267313385 [==========================>...] - ETA: 2s
244998144/267313385 [==========================>...] - ETA: 2s
245456896/267313385 [==========================>...] - ETA: 2s
245874688/267313385 [==========================>...] - ETA: 2s
246210560/267313385 [==========================>...] - ETA: 2s
246554624/267313385 [==========================>...] - ETA: 2s
246915072/267313385 [==========================>...] - ETA: 2s
247275520/267313385 [==========================>...] - ETA: 2s
247652352/267313385 [==========================>...] - ETA: 2s
247996416/267313385 [==========================>...] - ETA: 2s
248373248/267313385 [==========================>...] - ETA: 2s
248733696/267313385 [==========================>...] - ETA: 2s
249110528/267313385 [==========================>...] - ETA: 2s
249503744/267313385 [===========================>..] - ETA: 1s
249880576/267313385 [===========================>..] - ETA: 1s
250273792/267313385 [===========================>..] - ETA: 1s
250650624/267313385 [===========================>..] - ETA: 1s
251060224/267313385 [===========================>..] - ETA: 1s
251453440/267313385 [===========================>..] - ETA: 1s
251879424/267313385 [===========================>..] - ETA: 1s
252272640/267313385 [===========================>..] - ETA: 1s
252698624/267313385 [===========================>..] - ETA: 1s
253108224/267313385 [===========================>..] - ETA: 1s
253550592/267313385 [===========================>..] - ETA: 1s
253960192/267313385 [===========================>..] - ETA: 1s
254402560/267313385 [===========================>..] - ETA: 1s
254828544/267313385 [===========================>..] - ETA: 1s
255279104/267313385 [===========================>..] - ETA: 1s
255713280/267313385 [===========================>..] - ETA: 1s
256172032/267313385 [===========================>..] - ETA: 1s
256614400/267313385 [===========================>..] - ETA: 1s
257073152/267313385 [===========================>..] - ETA: 1s
257531904/267313385 [===========================>..] - ETA: 1s
258007040/267313385 [===========================>..] - ETA: 1s
258465792/267313385 [============================>.] - ETA: 0s
258859008/267313385 [============================>.] - ETA: 0s
259170304/267313385 [============================>.] - ETA: 0s
259547136/267313385 [============================>.] - ETA: 0s
259858432/267313385 [============================>.] - ETA: 0s
260251648/267313385 [============================>.] - ETA: 0s
260628480/267313385 [============================>.] - ETA: 0s
260956160/267313385 [============================>.] - ETA: 0s
261341184/267313385 [============================>.] - ETA: 0s
261693440/267313385 [============================>.] - ETA: 0s
262086656/267313385 [============================>.] - ETA: 0s
262447104/267313385 [============================>.] - ETA: 0s
262840320/267313385 [============================>.] - ETA: 0s
263217152/267313385 [============================>.] - ETA: 0s
263626752/267313385 [============================>.] - ETA: 0s
264003584/267313385 [============================>.] - ETA: 0s
264413184/267313385 [============================>.] - ETA: 0s
264806400/267313385 [============================>.] - ETA: 0s
265232384/267313385 [============================>.] - ETA: 0s
265641984/267313385 [============================>.] - ETA: 0s
266067968/267313385 [============================>.] - ETA: 0s
266493952/267313385 [============================>.] - ETA: 0s
266919936/267313385 [============================>.] - ETA: 0s
267313152/267313385 [============================>.] - ETA: 0s
267313385/267313385 [==============================] - 30s 0us/step

2. Load a pre-trained native Keras model

The model used in this example is AkidaUNet. It has an AkidaNet (0.5) backbone to extract features combined with a succession of separable transposed convolutional blocks to build an image segmentation map. A pre-trained floating point keras model is downloaded to save training time.

Note

  • The “transposed” convolutional feature is new in Akida 2.0.

  • The “separable transposed” operation is realized through the combination of a QuantizeML custom DepthwiseConv2DTranspose layer with a standard pointwise convolution.

The performance of the model is evaluated using both pixel accuracy and Binary IoU. The pixel accuracy describes how well the model can predict the segmentation mask pixel by pixel and the Binary IoU takes into account how close the predicted mask is to the ground truth.

from tensorflow.keras.utils import get_file
from akida_models.model_io import load_model

# Retrieve the model file from Brainchip data server
model_file = get_file("akida_unet_portrait128.h5",
                      "https://data.brainchip.com/models/AkidaV2/akida_unet/akida_unet_portrait128.h5",
                      cache_subdir='models')

# Load the native Keras pre-trained model
model_keras = load_model(model_file)
model_keras.summary()

Downloading data from https://data.brainchip.com/models/AkidaV2/akida_unet/akida_unet_portrait128.h5

   8192/4493952 [..............................] - ETA: 0s
 212992/4493952 [>.............................] - ETA: 1s
 704512/4493952 [===>..........................] - ETA: 0s
1122304/4493952 [======>.......................] - ETA: 0s
1540096/4493952 [=========>....................] - ETA: 0s
1941504/4493952 [===========>..................] - ETA: 0s
2367488/4493952 [==============>...............] - ETA: 0s
2809856/4493952 [=================>............] - ETA: 0s
3260416/4493952 [====================>.........] - ETA: 0s
3710976/4493952 [=======================>......] - ETA: 0s
4153344/4493952 [==========================>...] - ETA: 0s
4493952/4493952 [==============================] - 1s 0us/step
Model: "akida_unet"
_________________________________________________________________
 Layer (type)                Output Shape              Param #
=================================================================
 input (InputLayer)          [(None, 128, 128, 3)]     0

 rescaling (Rescaling)       (None, 128, 128, 3)       0

 conv_0 (Conv2D)             (None, 64, 64, 16)        432

 conv_0/BN (BatchNormalizati  (None, 64, 64, 16)       64
 on)

 conv_0/relu (ReLU)          (None, 64, 64, 16)        0

 conv_1 (Conv2D)             (None, 64, 64, 32)        4608

 conv_1/BN (BatchNormalizati  (None, 64, 64, 32)       128
 on)

 conv_1/relu (ReLU)          (None, 64, 64, 32)        0

 conv_2 (Conv2D)             (None, 32, 32, 64)        18432

 conv_2/BN (BatchNormalizati  (None, 32, 32, 64)       256
 on)

 conv_2/relu (ReLU)          (None, 32, 32, 64)        0

 conv_3 (Conv2D)             (None, 32, 32, 64)        36864

 conv_3/BN (BatchNormalizati  (None, 32, 32, 64)       256
 on)

 conv_3/relu (ReLU)          (None, 32, 32, 64)        0

 dw_separable_4 (DepthwiseCo  (None, 16, 16, 64)       576
 nv2D)

 pw_separable_4 (Conv2D)     (None, 16, 16, 128)       8192

 pw_separable_4/BN (BatchNor  (None, 16, 16, 128)      512
 malization)

 pw_separable_4/relu (ReLU)  (None, 16, 16, 128)       0

 dw_separable_5 (DepthwiseCo  (None, 16, 16, 128)      1152
 nv2D)

 pw_separable_5 (Conv2D)     (None, 16, 16, 128)       16384

 pw_separable_5/BN (BatchNor  (None, 16, 16, 128)      512
 malization)

 pw_separable_5/relu (ReLU)  (None, 16, 16, 128)       0

 dw_separable_6 (DepthwiseCo  (None, 8, 8, 128)        1152
 nv2D)

 pw_separable_6 (Conv2D)     (None, 8, 8, 256)         32768

 pw_separable_6/BN (BatchNor  (None, 8, 8, 256)        1024
 malization)

 pw_separable_6/relu (ReLU)  (None, 8, 8, 256)         0

 dw_separable_7 (DepthwiseCo  (None, 8, 8, 256)        2304
 nv2D)

 pw_separable_7 (Conv2D)     (None, 8, 8, 256)         65536

 pw_separable_7/BN (BatchNor  (None, 8, 8, 256)        1024
 malization)

 pw_separable_7/relu (ReLU)  (None, 8, 8, 256)         0

 dw_separable_8 (DepthwiseCo  (None, 8, 8, 256)        2304
 nv2D)

 pw_separable_8 (Conv2D)     (None, 8, 8, 256)         65536

 pw_separable_8/BN (BatchNor  (None, 8, 8, 256)        1024
 malization)

 pw_separable_8/relu (ReLU)  (None, 8, 8, 256)         0

 dw_separable_9 (DepthwiseCo  (None, 8, 8, 256)        2304
 nv2D)

 pw_separable_9 (Conv2D)     (None, 8, 8, 256)         65536

 pw_separable_9/BN (BatchNor  (None, 8, 8, 256)        1024
 malization)

 pw_separable_9/relu (ReLU)  (None, 8, 8, 256)         0

 dw_separable_10 (DepthwiseC  (None, 8, 8, 256)        2304
 onv2D)

 pw_separable_10 (Conv2D)    (None, 8, 8, 256)         65536

 pw_separable_10/BN (BatchNo  (None, 8, 8, 256)        1024
 rmalization)

 pw_separable_10/relu (ReLU)  (None, 8, 8, 256)        0

 dw_separable_11 (DepthwiseC  (None, 8, 8, 256)        2304
 onv2D)

 pw_separable_11 (Conv2D)    (None, 8, 8, 256)         65536

 pw_separable_11/BN (BatchNo  (None, 8, 8, 256)        1024
 rmalization)

 pw_separable_11/relu (ReLU)  (None, 8, 8, 256)        0

 dw_separable_12 (DepthwiseC  (None, 4, 4, 256)        2304
 onv2D)

 pw_separable_12 (Conv2D)    (None, 4, 4, 512)         131072

 pw_separable_12/BN (BatchNo  (None, 4, 4, 512)        2048
 rmalization)

 pw_separable_12/relu (ReLU)  (None, 4, 4, 512)        0

 dw_separable_13 (DepthwiseC  (None, 4, 4, 512)        4608
 onv2D)

 pw_separable_13 (Conv2D)    (None, 4, 4, 512)         262144

 pw_separable_13/BN (BatchNo  (None, 4, 4, 512)        2048
 rmalization)

 pw_separable_13/relu (ReLU)  (None, 4, 4, 512)        0

 dw_sepconv_t_0 (DepthwiseCo  (None, 8, 8, 512)        5120
 nv2DTranspose)

 pw_sepconv_t_0 (Conv2D)     (None, 8, 8, 256)         131328

 pw_sepconv_t_0/BN (BatchNor  (None, 8, 8, 256)        1024
 malization)

 pw_sepconv_t_0/relu (ReLU)  (None, 8, 8, 256)         0

 dropout (Dropout)           (None, 8, 8, 256)         0

 dw_sepconv_t_1 (DepthwiseCo  (None, 16, 16, 256)      2560
 nv2DTranspose)

 pw_sepconv_t_1 (Conv2D)     (None, 16, 16, 128)       32896

 pw_sepconv_t_1/BN (BatchNor  (None, 16, 16, 128)      512
 malization)

 pw_sepconv_t_1/relu (ReLU)  (None, 16, 16, 128)       0

 dropout_1 (Dropout)         (None, 16, 16, 128)       0

 dw_sepconv_t_2 (DepthwiseCo  (None, 32, 32, 128)      1280
 nv2DTranspose)

 pw_sepconv_t_2 (Conv2D)     (None, 32, 32, 64)        8256

 pw_sepconv_t_2/BN (BatchNor  (None, 32, 32, 64)       256
 malization)

 pw_sepconv_t_2/relu (ReLU)  (None, 32, 32, 64)        0

 dropout_2 (Dropout)         (None, 32, 32, 64)        0

 dw_sepconv_t_3 (DepthwiseCo  (None, 64, 64, 64)       640
 nv2DTranspose)

 pw_sepconv_t_3 (Conv2D)     (None, 64, 64, 32)        2080

 pw_sepconv_t_3/BN (BatchNor  (None, 64, 64, 32)       128
 malization)

 pw_sepconv_t_3/relu (ReLU)  (None, 64, 64, 32)        0

 dropout_3 (Dropout)         (None, 64, 64, 32)        0

 dw_sepconv_t_4 (DepthwiseCo  (None, 128, 128, 32)     320
 nv2DTranspose)

 pw_sepconv_t_4 (Conv2D)     (None, 128, 128, 16)      528

 pw_sepconv_t_4/BN (BatchNor  (None, 128, 128, 16)     64
 malization)

 pw_sepconv_t_4/relu (ReLU)  (None, 128, 128, 16)      0

 dropout_4 (Dropout)         (None, 128, 128, 16)      0

 head (Conv2D)               (None, 128, 128, 1)       17

 sigmoid_act (Activation)    (None, 128, 128, 1)       0

=================================================================
Total params: 1,058,865
Trainable params: 1,051,889
Non-trainable params: 6,976
_________________________________________________________________

from keras.metrics import BinaryIoU

# Compile the native Keras model (required to evaluate the metrics)
model_keras.compile(loss='binary_crossentropy', metrics=[BinaryIoU(), 'accuracy'])

# Check Keras model performance
_, biou, acc = model_keras.evaluate(x_val, y_val, steps=steps, verbose=0)

print(f"Keras binary IoU / pixel accuracy: {biou:.4f} / {100*acc:.2f}%")

Keras binary IoU / pixel accuracy: 0.9455 / 97.28%

3. Load a pre-trained quantized Keras model

The next step is to quantize and potentially perform Quantize Aware Training (QAT) on the Keras model from the previous step. After the Keras model is quantized to 8-bits for all weights and activations, QAT is used to maintain the performance of the quantized model. Again, a pre-trained model is downloaded to save runtime.

from akida_models import akida_unet_portrait128_pretrained

# Load the pre-trained quantized model
model_quantized_keras = akida_unet_portrait128_pretrained()
model_quantized_keras.summary()

Downloading data from https://data.brainchip.com/models/AkidaV2/akida_unet/akida_unet_portrait128_i8_w8_a8.h5.

      0/4520576 [..............................] - ETA: 0s
  98304/4520576 [..............................] - ETA: 2s
 385024/4520576 [=>............................] - ETA: 1s
 696320/4520576 [===>..........................] - ETA: 0s
1064960/4520576 [======>.......................] - ETA: 0s
1105920/4520576 [======>.......................] - ETA: 0s
1286144/4520576 [=======>......................] - ETA: 0s
1400832/4520576 [========>.....................] - ETA: 0s
1507328/4520576 [=========>....................] - ETA: 0s
1581056/4520576 [=========>....................] - ETA: 0s
1695744/4520576 [==========>...................] - ETA: 0s
1785856/4520576 [==========>...................] - ETA: 0s
1835008/4520576 [===========>..................] - ETA: 0s
1933312/4520576 [===========>..................] - ETA: 0s
2007040/4520576 [============>.................] - ETA: 0s
2039808/4520576 [============>.................] - ETA: 1s
2080768/4520576 [============>.................] - ETA: 1s
2129920/4520576 [=============>................] - ETA: 1s
2162688/4520576 [=============>................] - ETA: 1s
2203648/4520576 [=============>................] - ETA: 1s
2236416/4520576 [=============>................] - ETA: 1s
2269184/4520576 [==============>...............] - ETA: 1s
2301952/4520576 [==============>...............] - ETA: 1s
2318336/4520576 [==============>...............] - ETA: 1s
2351104/4520576 [==============>...............] - ETA: 1s
2375680/4520576 [==============>...............] - ETA: 1s
2392064/4520576 [==============>...............] - ETA: 1s
2408448/4520576 [==============>...............] - ETA: 1s
2433024/4520576 [===============>..............] - ETA: 1s
2457600/4520576 [===============>..............] - ETA: 1s
2482176/4520576 [===============>..............] - ETA: 2s
2506752/4520576 [===============>..............] - ETA: 2s
2531328/4520576 [===============>..............] - ETA: 2s
2564096/4520576 [================>.............] - ETA: 2s
2596864/4520576 [================>.............] - ETA: 2s
2637824/4520576 [================>.............] - ETA: 2s
2695168/4520576 [================>.............] - ETA: 1s
2752512/4520576 [=================>............] - ETA: 1s
2818048/4520576 [=================>............] - ETA: 1s
2891776/4520576 [==================>...........] - ETA: 1s
2990080/4520576 [==================>...........] - ETA: 1s
3096576/4520576 [===================>..........] - ETA: 1s
3219456/4520576 [====================>.........] - ETA: 1s
3358720/4520576 [=====================>........] - ETA: 1s
3522560/4520576 [======================>.......] - ETA: 0s
3710976/4520576 [=======================>......] - ETA: 0s
3915776/4520576 [========================>.....] - ETA: 0s
4161536/4520576 [==========================>...] - ETA: 0s
4456448/4520576 [============================>.] - ETA: 0s
4520576/4520576 [==============================] - 4s 1us/step
Model: "akida_unet"
_________________________________________________________________
 Layer (type)                Output Shape              Param #
=================================================================
 input (InputLayer)          [(None, 128, 128, 3)]     0

 rescaling (QuantizedRescali  (None, 128, 128, 3)      0
 ng)

 conv_0 (QuantizedConv2D)    (None, 64, 64, 16)        448

 conv_0/relu (QuantizedReLU)  (None, 64, 64, 16)       32

 conv_1 (QuantizedConv2D)    (None, 64, 64, 32)        4640

 conv_1/relu (QuantizedReLU)  (None, 64, 64, 32)       64

 conv_2 (QuantizedConv2D)    (None, 32, 32, 64)        18496

 conv_2/relu (QuantizedReLU)  (None, 32, 32, 64)       128

 conv_3 (QuantizedConv2D)    (None, 32, 32, 64)        36928

 conv_3/relu (QuantizedReLU)  (None, 32, 32, 64)       128

 dw_separable_4 (QuantizedDe  (None, 16, 16, 64)       704
 pthwiseConv2D)

 pw_separable_4 (QuantizedCo  (None, 16, 16, 128)      8320
 nv2D)

 pw_separable_4/relu (Quanti  (None, 16, 16, 128)      256
 zedReLU)

 dw_separable_5 (QuantizedDe  (None, 16, 16, 128)      1408
 pthwiseConv2D)

 pw_separable_5 (QuantizedCo  (None, 16, 16, 128)      16512
 nv2D)

 pw_separable_5/relu (Quanti  (None, 16, 16, 128)      256
 zedReLU)

 dw_separable_6 (QuantizedDe  (None, 8, 8, 128)        1408
 pthwiseConv2D)

 pw_separable_6 (QuantizedCo  (None, 8, 8, 256)        33024
 nv2D)

 pw_separable_6/relu (Quanti  (None, 8, 8, 256)        512
 zedReLU)

 dw_separable_7 (QuantizedDe  (None, 8, 8, 256)        2816
 pthwiseConv2D)

 pw_separable_7 (QuantizedCo  (None, 8, 8, 256)        65792
 nv2D)

 pw_separable_7/relu (Quanti  (None, 8, 8, 256)        512
 zedReLU)

 dw_separable_8 (QuantizedDe  (None, 8, 8, 256)        2816
 pthwiseConv2D)

 pw_separable_8 (QuantizedCo  (None, 8, 8, 256)        65792
 nv2D)

 pw_separable_8/relu (Quanti  (None, 8, 8, 256)        512
 zedReLU)

 dw_separable_9 (QuantizedDe  (None, 8, 8, 256)        2816
 pthwiseConv2D)

 pw_separable_9 (QuantizedCo  (None, 8, 8, 256)        65792
 nv2D)

 pw_separable_9/relu (Quanti  (None, 8, 8, 256)        512
 zedReLU)

 dw_separable_10 (QuantizedD  (None, 8, 8, 256)        2816
 epthwiseConv2D)

 pw_separable_10 (QuantizedC  (None, 8, 8, 256)        65792
 onv2D)

 pw_separable_10/relu (Quant  (None, 8, 8, 256)        512
 izedReLU)

 dw_separable_11 (QuantizedD  (None, 8, 8, 256)        2816
 epthwiseConv2D)

 pw_separable_11 (QuantizedC  (None, 8, 8, 256)        65792
 onv2D)

 pw_separable_11/relu (Quant  (None, 8, 8, 256)        512
 izedReLU)

 dw_separable_12 (QuantizedD  (None, 4, 4, 256)        2816
 epthwiseConv2D)

 pw_separable_12 (QuantizedC  (None, 4, 4, 512)        131584
 onv2D)

 pw_separable_12/relu (Quant  (None, 4, 4, 512)        1024
 izedReLU)

 dw_separable_13 (QuantizedD  (None, 4, 4, 512)        5632
 epthwiseConv2D)

 pw_separable_13 (QuantizedC  (None, 4, 4, 512)        262656
 onv2D)

 pw_separable_13/relu (Quant  (None, 4, 4, 512)        1024
 izedReLU)

 dw_sepconv_t_0 (QuantizedDe  (None, 8, 8, 512)        6144
 pthwiseConv2DTranspose)

 pw_sepconv_t_0 (QuantizedCo  (None, 8, 8, 256)        131328
 nv2D)

 pw_sepconv_t_0/relu (Quanti  (None, 8, 8, 256)        512
 zedReLU)

 dropout (QuantizedDropout)  (None, 8, 8, 256)         0

 dw_sepconv_t_1 (QuantizedDe  (None, 16, 16, 256)      3072
 pthwiseConv2DTranspose)

 pw_sepconv_t_1 (QuantizedCo  (None, 16, 16, 128)      32896
 nv2D)

 pw_sepconv_t_1/relu (Quanti  (None, 16, 16, 128)      256
 zedReLU)

 dropout_1 (QuantizedDropout  (None, 16, 16, 128)      0
 )

 dw_sepconv_t_2 (QuantizedDe  (None, 32, 32, 128)      1536
 pthwiseConv2DTranspose)

 pw_sepconv_t_2 (QuantizedCo  (None, 32, 32, 64)       8256
 nv2D)

 pw_sepconv_t_2/relu (Quanti  (None, 32, 32, 64)       128
 zedReLU)

 dropout_2 (QuantizedDropout  (None, 32, 32, 64)       0
 )

 dw_sepconv_t_3 (QuantizedDe  (None, 64, 64, 64)       768
 pthwiseConv2DTranspose)

 pw_sepconv_t_3 (QuantizedCo  (None, 64, 64, 32)       2080
 nv2D)

 pw_sepconv_t_3/relu (Quanti  (None, 64, 64, 32)       64
 zedReLU)

 dropout_3 (QuantizedDropout  (None, 64, 64, 32)       0
 )

 dw_sepconv_t_4 (QuantizedDe  (None, 128, 128, 32)     384
 pthwiseConv2DTranspose)

 pw_sepconv_t_4 (QuantizedCo  (None, 128, 128, 16)     528
 nv2D)

 pw_sepconv_t_4/relu (Quanti  (None, 128, 128, 16)     32
 zedReLU)

 dropout_4 (QuantizedDropout  (None, 128, 128, 16)     0
 )

 head (QuantizedConv2D)      (None, 128, 128, 1)       17

 dequantizer (Dequantizer)   (None, 128, 128, 1)       0

 sigmoid_act (Activation)    (None, 128, 128, 1)       0

=================================================================
Total params: 1,061,601
Trainable params: 1,047,905
Non-trainable params: 13,696
_________________________________________________________________

# Compile the quantized Keras model (required to evaluate the metrics)
model_quantized_keras.compile(loss='binary_crossentropy', metrics=[BinaryIoU(), 'accuracy'])

# Check Keras model performance
_, biou, acc = model_quantized_keras.evaluate(x_val, y_val, steps=steps, verbose=0)

print(f"Keras quantized binary IoU / pixel accuracy: {biou:.4f} / {100*acc:.2f}%")

Keras quantized binary IoU / pixel accuracy: 0.9399 / 97.01%

4. Conversion to Akida

Finally, the quantized Keras model from the previous step is converted into an Akida model and its performance is evaluated. Note that the original performance of the keras floating point model is maintained throughout the conversion process in this example.

from cnn2snn import convert

# Convert the model
model_akida = convert(model_quantized_keras)
model_akida.summary()

                  Model Summary
_________________________________________________
Input shape    Output shape   Sequences  Layers
=================================================
[128, 128, 3]  [128, 128, 1]  1          36
_________________________________________________

_____________________________________________________________________________
Layer (type)                               Output shape    Kernel shape

====================== SW/conv_0-dequantizer (Software) =====================

conv_0 (InputConv2D)                       [64, 64, 16]    (3, 3, 3, 16)
_____________________________________________________________________________
conv_1 (Conv2D)                            [64, 64, 32]    (3, 3, 16, 32)
_____________________________________________________________________________
conv_2 (Conv2D)                            [32, 32, 64]    (3, 3, 32, 64)
_____________________________________________________________________________
conv_3 (Conv2D)                            [32, 32, 64]    (3, 3, 64, 64)
_____________________________________________________________________________
dw_separable_4 (DepthwiseConv2D)           [16, 16, 64]    (3, 3, 64, 1)
_____________________________________________________________________________
pw_separable_4 (Conv2D)                    [16, 16, 128]   (1, 1, 64, 128)
_____________________________________________________________________________
dw_separable_5 (DepthwiseConv2D)           [16, 16, 128]   (3, 3, 128, 1)
_____________________________________________________________________________
pw_separable_5 (Conv2D)                    [16, 16, 128]   (1, 1, 128, 128)
_____________________________________________________________________________
dw_separable_6 (DepthwiseConv2D)           [8, 8, 128]     (3, 3, 128, 1)
_____________________________________________________________________________
pw_separable_6 (Conv2D)                    [8, 8, 256]     (1, 1, 128, 256)
_____________________________________________________________________________
dw_separable_7 (DepthwiseConv2D)           [8, 8, 256]     (3, 3, 256, 1)
_____________________________________________________________________________
pw_separable_7 (Conv2D)                    [8, 8, 256]     (1, 1, 256, 256)
_____________________________________________________________________________
dw_separable_8 (DepthwiseConv2D)           [8, 8, 256]     (3, 3, 256, 1)
_____________________________________________________________________________
pw_separable_8 (Conv2D)                    [8, 8, 256]     (1, 1, 256, 256)
_____________________________________________________________________________
dw_separable_9 (DepthwiseConv2D)           [8, 8, 256]     (3, 3, 256, 1)
_____________________________________________________________________________
pw_separable_9 (Conv2D)                    [8, 8, 256]     (1, 1, 256, 256)
_____________________________________________________________________________
dw_separable_10 (DepthwiseConv2D)          [8, 8, 256]     (3, 3, 256, 1)
_____________________________________________________________________________
pw_separable_10 (Conv2D)                   [8, 8, 256]     (1, 1, 256, 256)
_____________________________________________________________________________
dw_separable_11 (DepthwiseConv2D)          [8, 8, 256]     (3, 3, 256, 1)
_____________________________________________________________________________
pw_separable_11 (Conv2D)                   [8, 8, 256]     (1, 1, 256, 256)
_____________________________________________________________________________
dw_separable_12 (DepthwiseConv2D)          [4, 4, 256]     (3, 3, 256, 1)
_____________________________________________________________________________
pw_separable_12 (Conv2D)                   [4, 4, 512]     (1, 1, 256, 512)
_____________________________________________________________________________
dw_separable_13 (DepthwiseConv2D)          [4, 4, 512]     (3, 3, 512, 1)
_____________________________________________________________________________
pw_separable_13 (Conv2D)                   [4, 4, 512]     (1, 1, 512, 512)
_____________________________________________________________________________
dw_sepconv_t_0 (DepthwiseConv2DTranspose)  [8, 8, 512]     (3, 3, 512, 1)
_____________________________________________________________________________
pw_sepconv_t_0 (Conv2D)                    [8, 8, 256]     (1, 1, 512, 256)
_____________________________________________________________________________
dw_sepconv_t_1 (DepthwiseConv2DTranspose)  [16, 16, 256]   (3, 3, 256, 1)
_____________________________________________________________________________
pw_sepconv_t_1 (Conv2D)                    [16, 16, 128]   (1, 1, 256, 128)
_____________________________________________________________________________
dw_sepconv_t_2 (DepthwiseConv2DTranspose)  [32, 32, 128]   (3, 3, 128, 1)
_____________________________________________________________________________
pw_sepconv_t_2 (Conv2D)                    [32, 32, 64]    (1, 1, 128, 64)
_____________________________________________________________________________
dw_sepconv_t_3 (DepthwiseConv2DTranspose)  [64, 64, 64]    (3, 3, 64, 1)
_____________________________________________________________________________
pw_sepconv_t_3 (Conv2D)                    [64, 64, 32]    (1, 1, 64, 32)
_____________________________________________________________________________
dw_sepconv_t_4 (DepthwiseConv2DTranspose)  [128, 128, 32]  (3, 3, 32, 1)
_____________________________________________________________________________
pw_sepconv_t_4 (Conv2D)                    [128, 128, 16]  (1, 1, 32, 16)
_____________________________________________________________________________
head (Conv2D)                              [128, 128, 1]   (1, 1, 16, 1)
_____________________________________________________________________________
dequantizer (Dequantizer)                  [128, 128, 1]   N/A
_____________________________________________________________________________

import tensorflow as tf

# Check Akida model performance
labels, pots = None, None

for s in range(steps):
    batch = x_val[s * batch_size: (s + 1) * batch_size, :]
    label_batch = y_val[s * batch_size: (s + 1) * batch_size, :]
    pots_batch = model_akida.predict(batch.astype('uint8'))

    if labels is None:
        labels = label_batch
        pots = pots_batch
    else:
        labels = np.concatenate((labels, label_batch))
        pots = np.concatenate((pots, pots_batch))
preds = tf.keras.activations.sigmoid(pots)

m_binary_iou = tf.keras.metrics.BinaryIoU(target_class_ids=[0, 1], threshold=0.5)
m_binary_iou.update_state(labels, preds)
binary_iou = m_binary_iou.result().numpy()

m_accuracy = tf.keras.metrics.Accuracy()
m_accuracy.update_state(labels, preds > 0.5)
accuracy = m_accuracy.result().numpy()
print(f"Akida binary IoU / pixel accuracy: {binary_iou:.4f} / {100*accuracy:.2f}%")

# For non-regression purpose
assert binary_iou > 0.9

Akida binary IoU / pixel accuracy: 0.9388 / 97.01%

5. Segment a single image

For visualization of the person segmentation performed by the Akida model, display a single image along with the segmentation produced by the original floating point model and the ground truth segmentation.

import matplotlib.pyplot as plt

# Estimate age on a random single image and display Keras and Akida outputs
sample = np.expand_dims(x_val[id, :], 0)
keras_out = model_keras(sample)
akida_out = tf.keras.activations.sigmoid(model_akida.forward(sample.astype('uint8')))

fig, axs = plt.subplots(1, 3, constrained_layout=True)
axs[0].imshow(keras_out[0] * sample[0] / 255.)
axs[0].set_title('Keras segmentation', fontsize=10)
axs[0].axis('off')

axs[1].imshow(akida_out[0] * sample[0] / 255.)
axs[1].set_title('Akida segmentation', fontsize=10)
axs[1].axis('off')

axs[2].imshow(y_val[id] * sample[0] / 255.)
axs[2].set_title('Expected segmentation', fontsize=10)
axs[2].axis('off')

plt.show()
Keras segmentation, Akida segmentation, Expected segmentation

Total running time of the script: (1 minutes 45.930 seconds)

Gallery generated by Sphinx-Gallery