Tensorflow inference using Java API extremely slow

I downloaded the python3 example for DeepLabv3 inference which uses a pre-trained model. The runtime for the actual inference is about 19 seconds on the CPU I'm using. Tensorflow was installes with pip:

pip install intel-tensorflow

This is the code from the colab Jupyter notebook:

#!/usr/bin/python

import os
from io import BytesIO
import tarfile
import tempfile
from six.moves import urllib

from matplotlib import gridspec
from matplotlib import pyplot as plt
import numpy as np
from PIL import Image
from timeit import default_timer as timer

#%tensorflow_version 1.x
import tensorflow.compat.v1 as tf
#import tensorflow as tf

class DeepLabModel(object):
  """Class to load deeplab model and run inference."""

  INPUT_TENSOR_NAME = 'ImageTensor:0'
  OUTPUT_TENSOR_NAME = 'SemanticPredictions:0'
  INPUT_SIZE = 513
  FROZEN_GRAPH_NAME = 'frozen_inference_graph'

  def __init__(self, tarball_path):
    """Creates and loads pretrained deeplab model."""
    self.graph = tf.Graph()

    graph_def = None
    # Extract frozen graph from tar archive.
    tar_file = tarfile.open(tarball_path)
    for tar_info in tar_file.getmembers():
      if self.FROZEN_GRAPH_NAME in os.path.basename(tar_info.name):
        file_handle = tar_file.extractfile(tar_info)
        graph_def = tf.GraphDef.FromString(file_handle.read())
        break

    tar_file.close()

    if graph_def is None:
      raise RuntimeError('Cannot find inference graph in tar archive.')

    with self.graph.as_default():
      tf.import_graph_def(graph_def, name='')

    self.sess = tf.Session(graph=self.graph)

  def run(self, image):
    """Runs inference on a single image.

    Args:
      image: A PIL.Image object, raw input image.

    Returns:
      resized_image: RGB image resized from original input image.
      seg_map: Segmentation map of `resized_image`.
    """
    width, height = image.size
    resize_ratio = 1.0 * self.INPUT_SIZE / max(width, height)
    target_size = (int(resize_ratio * width), int(resize_ratio * height))
    resized_image = image.convert('RGB').resize(target_size, Image.ANTIALIAS)
    start = timer()
    batch_seg_map = self.sess.run(
        self.OUTPUT_TENSOR_NAME,
        feed_dict={self.INPUT_TENSOR_NAME: [np.asarray(resized_image)]})
    end = timer()
    print("Inference duration: ", end-start)
    seg_map = batch_seg_map[0]
    return resized_image, seg_map


def create_pascal_label_colormap():
  """Creates a label colormap used in PASCAL VOC segmentation benchmark.

  Returns:
    A Colormap for visualizing segmentation results.
  """
  colormap = np.zeros((256, 3), dtype=int)
  ind = np.arange(256, dtype=int)

  for shift in reversed(range(8)):
    for channel in range(3):
      colormap[:, channel] |= ((ind >> channel) & 1) << shift
    ind >>= 3

  return colormap


def label_to_color_image(label):
  """Adds color defined by the dataset colormap to the label.

  Args:
    label: A 2D array with integer type, storing the segmentation label.

  Returns:
    result: A 2D array with floating type. The element of the array
      is the color indexed by the corresponding element in the input label
      to the PASCAL color map.

  Raises:
    ValueError: If label is not of rank 2 or its value is larger than color
      map maximum entry.
  """
  if label.ndim != 2:
    raise ValueError('Expect 2-D input label')

  colormap = create_pascal_label_colormap()

  if np.max(label) >= len(colormap):
    raise ValueError('label value too large.')

  return colormap[label]


def vis_segmentation(image, seg_map):
  """Visualizes input image, segmentation map and overlay view."""
  plt.figure(figsize=(15, 5))
  grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1])

  plt.subplot(grid_spec[0])
  plt.imshow(image)
  plt.axis('off')
  plt.title('input image')

  plt.subplot(grid_spec[1])
  seg_image = label_to_color_image(seg_map).astype(np.uint8)
  plt.imshow(seg_image)
  plt.axis('off')
  plt.title('segmentation map')

  plt.subplot(grid_spec[2])
  plt.imshow(image)
  plt.imshow(seg_image, alpha=0.7)
  plt.axis('off')
  plt.title('segmentation overlay')

  unique_labels = np.unique(seg_map)
  ax = plt.subplot(grid_spec[3])
  plt.imshow(
      FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation='nearest')
  ax.yaxis.tick_right()
  plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels])
  plt.xticks([], [])
  ax.tick_params(width=0.0)
  plt.grid('off')
  plt.show()


LABEL_NAMES = np.asarray([
    'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus',
    'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike',
    'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tv'
])

FULL_LABEL_MAP = np.arange(len(LABEL_NAMES)).reshape(len(LABEL_NAMES), 1)
FULL_COLOR_MAP = label_to_color_image(FULL_LABEL_MAP)

MODEL_NAME = 'xception_coco_voctrainval'  # @param ['mobilenetv2_coco_voctrainaug', 'mobilenetv2_coco_voctrainval', 'xception_coco_voctrainaug', 'xception_coco_voctrainval']

_DOWNLOAD_URL_PREFIX = 'http://download.tensorflow.org/models/'
_MODEL_URLS = {
    'mobilenetv2_coco_voctrainaug':
        'deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz',
    'mobilenetv2_coco_voctrainval':
        'deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz',
    'xception_coco_voctrainaug':
        'deeplabv3_pascal_train_aug_2018_01_04.tar.gz',
    'xception_coco_voctrainval':
        'deeplabv3_pascal_trainval_2018_01_04.tar.gz',
}
_TARBALL_NAME = 'deeplab_model.tar.gz'

model_dir = 'model'
tf.io.gfile.makedirs(model_dir)

download_path = os.path.join(model_dir, _TARBALL_NAME)
print('downloading model, this might take a while...')
urllib.request.urlretrieve(_DOWNLOAD_URL_PREFIX + _MODEL_URLS[MODEL_NAME],
                   download_path)
print('download completed! loading DeepLab model...')

MODEL = DeepLabModel(download_path)
print('model loaded successfully!')

SAMPLE_IMAGE = 'image1'  # @param ['image1', 'image2', 'image3']
IMAGE_URL = 'file:///home/rhobincu/man-in-white-dress-shirt-sitting-on-black-rolling-chair-840996.jpg'  #@param {type:"string"}

_SAMPLE_URL = ('https://github.com/tensorflow/models/blob/master/research/'
               'deeplab/g3doc/img/%s.jpg?raw=true')


def run_visualization(url):
  """Inferences DeepLab model and visualizes result."""
  try:
    f = urllib.request.urlopen(url)
    jpeg_str = f.read()
    original_im = Image.open(BytesIO(jpeg_str))
  except IOError:
    print('Cannot retrieve image. Please check url: ' + url)
    return

  print('running deeplab on image %s...' % url)
  resized_im, seg_map = MODEL.run(original_im)

  vis_segmentation(resized_im, seg_map)


image_url = IMAGE_URL or _SAMPLE_URL % SAMPLE_IMAGE
run_visualization(image_url)

With output:

rhobincu@ml:~/gitroot/test$ ./test.py 
downloading model, this might take a while...
download completed! loading DeepLab model...
2020-04-08 14:51:24.066757: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2199980000 Hz
2020-04-08 14:51:24.080415: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5561af0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2020-04-08 14:51:24.080567: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version
2020-04-08 14:51:24.081792: I tensorflow/core/common_runtime/process_util.cc:147] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.
model loaded successfully!
running deeplab on image file:///home/rhobincu/man-in-white-dress-shirt-sitting-on-black-rolling-chair-840996.jpg...
Inferrence duration:  18.454864561999784

I have attempted to re-write this in Java. I have compiled tensorflow from sources by cloning https://github.com/tensorflow/tensorflow tag v2.1.0 and running

bazel build -c opt --copt=-mavx --copt=-msse2 --copt=-msse3 --copt=-msse4.1 --copt=-msse4.2 --copt=-mfpmath=both //tensorflow:install_headers //tensorflow:libtensorflow_cc.so //tensorflow:libtensorflow_framework.so //tensorflow/java:tensorflow  //tensorflow/java:libtensorflow_jni

The following is the corresponding Java code:

package tensorflowapp;

import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import org.opencv.core.Mat;
import org.opencv.core.Size;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import org.tensorflow.DataType;
import org.tensorflow.Graph;
import org.tensorflow.Output;
import org.tensorflow.Session;
import org.tensorflow.Tensor;
import org.tensorflow.TensorFlow;
import org.tensorflow.types.UInt8;

/**
 * Sample use of the TensorFlow Java API to label images using a pre-trained
 * model.
 */
public class LabelImage {

    static {
        System.load("/usr/local/share/java/opencv4/libopencv_java420.so");
	System.load("/opt/tensorflow/java/native/libtensorflow_jni.so");
    }

    static Session loadDeeplabModel() throws IOException {
        Graph graph = new Graph();
        graph.importGraphDef(Files.readAllBytes(Paths.get("model/deeplabv3_pascal_trainval/frozen_inference_graph.pb")));
        Session session = new Session(graph);
        return session;
    }

    static Tensor<UInt8> matToTensor(Mat image) {
        byte[] byteData = new byte[(int) image.total() * image.channels()];
        image.get(0, 0, byteData);
        return Tensor.create(UInt8.class, new long[]{1, 1, image.width() * image.height(), 3}, ByteBuffer.wrap(byteData));
    }

    public static void main(String[] args) throws IOException {
        Session session = loadDeeplabModel();
        Mat image = Imgcodecs.imread(args[0], Imgcodecs.IMREAD_COLOR);
        Mat resized = new Mat();
        double scale = 513.0 / Math.max(image.width(), image.height());
        Size destinationSize = new Size(image.width() * scale, image.height() * scale);
        System.out.println("Destination size: " + destinationSize);
        Imgproc.resize(image, resized, destinationSize);
        Tensor<UInt8> imageTensor = matToTensor(resized);
        
        List<Tensor<?>> result = session.runner().feed("ImageTensor:0", imageTensor).fetch("SemanticPredictions:0").run();//.get(0).expect(Float.class);
        System.out.println("Done");
    }

}

Running the following command:

time java -cp /opt/tensorflow/java/*:dist/TensorFlowApp.jar:/usr/local/share/java/opencv4/opencv-420.jar tensorflowapp.LabelImage ../../man-in-white-dress-shirt-sitting-on-black-rolling-chair-840996.jpg

Yields the following output:

2020-04-08 13:26:14.611201: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2199980000 Hz
2020-04-08 13:26:14.626568: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7f7038dea6d0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2020-04-08 13:26:14.626612: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version
Destination size: 513x342
2020-04-08 13:46:59.913359: W tensorflow/core/framework/op_kernel.cc:1655] OP_REQUIRES failed at spacetobatch_op.cc:219 : Invalid argument: padded_shape[1]=21942 is not divisible by block_shape[1]=4
Exception in thread "main" java.lang.IllegalArgumentException: padded_shape[1]=21942 is not divisible by block_shape[1]=4
[[{{node xception_65/exit_flow/block2/unit_1/xception_module/separable_conv1_depthwise/depthwise/SpaceToBatchND}}]]
at org.tensorflow.Session.run(Native Method)
at org.tensorflow.Session.access$100(Session.java:48)
at org.tensorflow.Session$Runner.runHelper(Session.java:326)
at org.tensorflow.Session$Runner.run(Session.java:276)
at tensorflowapp.LabelImage.main(LabelImage.java:58)
Command exited with non-zero status 1
21166.66user 3912.49system 20:48.87elapsed 2008%CPU (0avgtext+0avgdata 27929748maxresident)k
0inputs+408outputs (0major+269297302minor)pagefaults 0swaps

Besides the error itself, the runtime is 3912 seconds...

</details>
# 答案1
**得分**: 1
关于推断时间，你是否尝试过使用相同的会话再次运行它？TensorFlow可能会在第一次运行时惰性地初始化一些资源，因此您可能希望为所有其他推断运行保留相同的会话，而不是为每个推断运行创建一个新会话。
一种常见做法是在进行实际推断之前，先用虚拟运行进行[预热](https://www.tensorflow.org/tfx/serving/saved_model_warmup)（链接只展示了TFX如何实现，但对于Java而言原理相同）。
<details>
<summary>英文:</summary>
For the inference time, did you tried to run it a second time using the same session? TensorFlow can initialize some resources lazily on the first run, so you might want to keep that same session available for all other inference runs as well instead of creating a new one for each of them.
A common practice is to [warm it up](https://www.tensorflow.org/tfx/serving/saved_model_warmup) once with a dummy run before doing real inference *(the link just shows how TFX does it but it’s the same principle for Java)*.
</details>