Input 0 is incompatible with layer when using dataset.batch

huangapple
go评论92阅读模式
英文:

Input 0 is incompatible with layer when using dataset.batch

问题

以下是您提供的代码的翻译部分：

  1. 我提供了以下正常运行的代码。
  3. import tensorflow as tf
  4. import numpy as np
  5. from tensorflow.keras.layers import Input, Dense, Reshape, Dropout, \
  6.     BatchNormalization, Activation, Conv2D, Conv2DTranspose, LeakyReLU
  7. from tensorflow.keras.models import Model
  9. AUTOTUNE = tf.data.experimental.AUTOTUNE
  10. HEIGHT = 39
  11. WIDTH = 39
  12. CHANNELS = 2
  13. SCALE_FACTOR = 4
  14. VAL_SPLIT = 0.1
  15. TRAIN_SPLIT = 0.8
  16. TEST_SPLIT = 0.1
  17. SEED = 1
  18. BUFFER_SIZE = 100
  19. LR = 1e-4
  20. BATCH_SIZE = 2
  22. INP_LOW = (HEIGHT, WIDTH, CHANNELS)
  24. gpus = tf.config.experimental.list_physical_devices('GPU')
  25. for gpu in gpus:
  26.     tf.config.experimental.set_memory_growth(gpu, True)
  29. def resize_and_rescale(low, high):
  31.     high = tf.image.resize(high, 
  32.                            (HEIGHT, WIDTH),
  33.                            preserve_aspect_ratio=False)
  34.     return low, high
  37. def split_train_test_val(ds,
  38.                          seed,
  39.                          train_split=TRAIN_SPLIT, 
  40.                          val_split=VAL_SPLIT, 
  41.                          test_split=TEST_SPLIT,
  42.                          shuffle=True, 
  43.                          shuffle_size=BUFFER_SIZE):                       
  44.     assert (train_split + test_split + val_split) == 1
  45.     
  46.     ds_size = len(ds)
  47.     if shuffle:
  48.         ds = ds.shuffle(shuffle_size, 
  49.                         reshuffle_each_iteration=False,
  50.                         seed=seed)
  51.     
  52.     train_size = int(train_split * ds_size)
  53.     val_size = int(val_split * ds_size)
  54.     test_size = int(test_split * ds_size)
  55.     
  56.     train_ds = ds.take(train_size)
  57.     test_ds = ds.skip(train_size)
  58.     val_ds = test_ds.skip(test_size)
  59.     test_ds = test_ds.take(test_size)
  60.     
  61.     return train_ds, val_ds, test_ds
  63. def prepare(ds, shuffle=False):
  64.     
  65.     ds = ds.map(resize_and_rescale, num_parallel_calls=AUTOTUNE)
  66.     ds = ds.cache()
  67.     if shuffle:
  68.         ds = ds.shuffle(buffer_size=BUFFER_SIZE)
  69.   
  70.     # ds = ds.batch(BATCH_SIZE)
  71.     # ds = ds.prefetch(buffer_size=AUTOTUNE)
  72.    
  73.     return ds
  75. def data_gen(low_res, high_res):
  76.     
  77.     dataset_low = tf.data.Dataset.from_tensor_slices(low_res)
  78.     dataset_high = tf.data.Dataset.from_tensor_slices(high_res)
  79.     
  80.     dataset = tf.data.Dataset.zip((dataset_low, dataset_high))
  81.     
  82.     train_ds, val_ds, test_ds = split_train_test_val(dataset,
  83.                                                      SEED,
  84.                                                      train_split=TRAIN_SPLIT, 
  85.                                                      val_split=VAL_SPLIT, 
  86.                                                      test_split=TEST_SPLIT,
  87.                                                      shuffle=True, 
  88.                                                      shuffle_size=BUFFER_SIZE)
  90.    
  91.     train_ds = prepare(train_ds, shuffle=True)
  92.     val_ds = prepare(val_ds)
  93.     test_ds = prepare(val_ds)
  94.     return train_ds, val_ds, test_ds
  97. def build_model(lr):
  98.     inp = Input(lr)
  99.     x = Dense(16)(inp)
  100.     x = Conv2DTranspose(CHANNELS, kernel_size=3, strides=1, padding='same')(x)
  101.     output = Activation('tanh')(x)
  102.     model = Model(inp, output)
  103.     return model
  106. low = np.load('low.npy')
  107. high = np.load('high.npy')
  109. train_ds, val_ds, test_ds = data_gen(low, high)
  111. model = build_model(INP_LOW)
  112. model.compile(loss=['mse'],
  113.               optimizer= tf.keras.optimizers.Adam(learning_rate=LR))
  115. train_low, train_high = tf.data.experimental.get_single_element(train_ds.batch(len(train_ds)))
  117. history = model.fit(train_low,
  118.                     train_high,
  119.                     epochs=2,
  120.                     batch_size=BATCH_SIZE)

请注意，我只提供了代码的翻译，不包括任何其他内容。如果您需要进一步的解释或指导，请告诉我。

英文:

I gave the following code which runs fine.

  1. import tensorflow as tf
  2. import numpy as np
  3. from tensorflow.keras.layers import Input, Dense, Reshape, Dropout, \
  4. BatchNormalization, Activation, Conv2D, Conv2DTranspose, LeakyReLU
  5. from tensorflow.keras.models import Model
  6. AUTOTUNE = tf.data.experimental.AUTOTUNE
  7. HEIGHT = 39
  8. WIDTH = 39
  9. CHANNELS = 2
  10. SCALE_FACTOR = 4
  11. VAL_SPLIT = 0.1
  12. TRAIN_SPLIT = 0.8
  13. TEST_SPLIT = 0.1
  14. SEED = 1
  15. BUFFER_SIZE = 100
  16. LR = 1e-4
  17. BATCH_SIZE = 2
  18. INP_LOW = (HEIGHT, WIDTH, CHANNELS)
  19. gpus = tf.config.experimental.list_physical_devices('GPU')
  20. for gpu in gpus:
  21. tf.config.experimental.set_memory_growth(gpu, True)
  22. def resize_and_rescale(low, high):
  23. high = tf.image.resize(high, 
  24. (HEIGHT, WIDTH),
  25. preserve_aspect_ratio=False)
  26. return low, high
  27. def split_train_test_val(ds,
  28. seed,
  29. train_split=TRAIN_SPLIT, 
  30. val_split=VAL_SPLIT, 
  31. test_split=TEST_SPLIT,
  32. shuffle=True, 
  33. shuffle_size=BUFFER_SIZE):                       
  34. assert (train_split + test_split + val_split) == 1
  35. ds_size = len(ds)
  36. if shuffle:
  37. ds = ds.shuffle(shuffle_size, 
  38. reshuffle_each_iteration=False,
  39. seed=seed)
  40. train_size = int(train_split * ds_size)
  41. val_size = int(val_split * ds_size)
  42. test_size = int(test_split * ds_size)
  43. train_ds = ds.take(train_size)
  44. test_ds = ds.skip(train_size)
  45. val_ds = test_ds.skip(test_size)
  46. test_ds = test_ds.take(test_size)
  47. return train_ds, val_ds, test_ds
  48. def prepare(ds, shuffle=False):
  49. ds = ds.map(resize_and_rescale, num_parallel_calls=AUTOTUNE)
  50. ds = ds.cache()
  51. if shuffle:
  52. ds = ds.shuffle(buffer_size=BUFFER_SIZE)
  53. #ds = ds.batch(BATCH_SIZE)
  54. #ds = ds.prefetch(buffer_size=AUTOTUNE)
  55. return ds
  56. def data_gen(low_res, high_res):
  57. dataset_low = tf.data.Dataset.from_tensor_slices(low_res)
  58. dataset_high = tf.data.Dataset.from_tensor_slices(high_res)
  59. dataset = tf.data.Dataset.zip((dataset_low, dataset_high))
  60. train_ds, val_ds, test_ds = split_train_test_val(dataset,
  61. SEED,
  62. train_split=TRAIN_SPLIT, 
  63. val_split=VAL_SPLIT, 
  64. test_split=TEST_SPLIT,
  65. shuffle=True, 
  66. shuffle_size=BUFFER_SIZE)
  67. train_ds = prepare(train_ds, shuffle=True)
  68. val_ds = prepare(val_ds)
  69. test_ds = prepare(val_ds)
  70. return train_ds, val_ds, test_ds
  71. def build_model(lr):
  72. inp = Input(lr)
  73. x = Dense(16)(inp)
  74. x = Conv2DTranspose(CHANNELS, kernel_size=3, strides=1, padding='same')(x)
  75. output = Activation('tanh')(x)
  76. model = Model(inp, output)
  77. return model
  78. low = np.load('low.npy')
  79. high = np.load('high.npy')
  80. train_ds, val_ds, test_ds = data_gen(low, high)
  81. model = build_model(INP_LOW)
  82. model.compile(loss=['mse'],
  83. optimizer= tf.keras.optimizers.Adam(learning_rate=LR))
  84. train_low, train_high = tf.data.experimental.get_single_element(train_ds.batch(len(train_ds)))
  85. history = model.fit(train_low,
  86. train_high,
  87. epochs=2,
  88. batch_size=BATCH_SIZE)

But when I try to use :

  1.  ds = ds.batch(BATCH_SIZE)
  2. ds = ds.prefetch(buffer_size=AUTOTUNE)

in the prepare function and at the same time, I ommit the batch_size in fit:

  1. history = model.fit(train_low,
  2. train_high,
  3. epochs=2)

I am receiving:

ValueError: Input 0 is incompatible with layer model_2: expected shape=(None, 39, 39, 2), found shape=(32, 2, 39, 39, 2)

You can find the data here

I would expect, since I remove the batch size from fit , to work.

答案1

得分: 1

在使用 tf.data.Dataset 在 model.fit 中时，只需提供 model.fit 的 x 参数，假设您的 tf.data.Dataset 返回一个元组 (input_features, targets)。

您可以在 keras.Model.fit 文档 中阅读更多信息，以下是相关信息的摘录：

> 参数
>
> - x: 输入数据。它可以是：
> - 一个 tf.data 数据集。应返回一个元组，要么是 (inputs, targets)，要么是 (inputs, targets, sample_weights)。

> - y: 如果 x 是一个数据集、生成器或 keras.utils.Sequence 实例，不应指定 y（因为目标将从 x 获取）。

假设 train_high 是您的输入特征，train_low 是您的目标，您只需调用 model.fit(train_ds, epochs=2)，并跳过以下这行代码：

  1. train_low, train_high = tf.data.experimental.get_single_element(train_ds.batch(len(train_ds)))
英文:

When using a tf.data.Dataset in model.fit, you should provide only the x argument of model.fit, with the assumption that your tf.data.Dataset returns a tuple (input_features, targets).

You can read more in the documentation of keras.Model.fit. Here's an excerpt with the relevant info:

> Args
>
> - x: Input data. It could be:
> - A tf.data dataset. Should return a tuple of either (inputs, targets) or (inputs, targets, sample_weights).
>
> - y: If x is a dataset, generator, or keras.utils.Sequence instance, y
> should not be specified (since targets will be obtained from x).

Assuming that train_high are your input features, and train_low are your targets, you should simply call model.fit(train_ds, epochs=2), and skip the line

  1. train_low, train_high = tf.data.experimental.get_single_element(train_ds.batch(len(train_ds)))

huangapple
  • 本文由 发表于 2023年5月17日 18:24:23
  • 转载请务必保留本文链接：https://go.coder-hub.com/76271075.html
  • deep-learning
  • python-3.x
  • tensorflow
  • tensorflow2.0
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定