class LuminosityAnalyzer : ImageAnalysis.Analyzer {
    private var lastTimeStamp = 0L
    private val TAG = this.javaClass.simpleName
    var luma = BehaviorSubject.create<Double>()
    override fun analyze(image: ImageProxy, rotationDegrees: Int) {
        val currentTimeStamp = System.currentTimeMillis()
        val intervalInSeconds = TimeUnit.SECONDS.toMillis(1)
        val deltaTime = currentTimeStamp - lastTimeStamp
        if(deltaTime >= intervalInSeconds) {
            val buffer = image.planes[0].buffer
            val data = buffer.toByteArray()
            val pixels = data.map { it.toInt() and 0xFF }
            luma.onNext(pixels.average())
            lastTimeStamp = currentTimeStamp
            Log.d(TAG, "Average luminosity: ${luma.value}")
        }
    }
    private fun ByteBuffer.toByteArray(): ByteArray {
        rewind()
        val data = ByteArray(remaining())
        get(data)
        return data
    }
}

/* display the luminosity */
private fun createLuminosityAnalyzer(): ImageAnalysis{
    val analyzerConfig = ImageAnalysisConfig.Builder().apply {
        setLensFacing(lensFacing)
        setImageReaderMode(ImageAnalysis.ImageReaderMode.ACQUIRE_LATEST_IMAGE)
    }.build()
    val analyzer = ImageAnalysis(analyzerConfig).apply {
        val luminosityAnalyzer = LuminosityAnalyzer()
        luminosityAnalyzer.luma
            .observeOn(AndroidSchedulers.mainThread())
            .subscribe({
            // success
            luminosity.text = it.toString()
        },{
            // error
            Log.d(TAG, "Can not get luminosity :(")
        })
        setAnalyzer(executor, luminosityAnalyzer)
    }
    return analyzer
}

So I figured out how to do it by myself
**Color Analyzer Class:**
class ColorAnalyzer : ImageAnalysis.Analyzer {
    private var lastTimeStamp = 0L
    private val TAG = this.javaClass.simpleName
    var hexColor = BehaviorSubject.create<Any>()
    /* every 100ms, analyze the image we receive from the camera */
    override fun analyze(image: ImageProxy, rotationDegrees: Int) {
        val currentTimeStamp = System.currentTimeMillis()
        val intervalInMilliSeconds = TimeUnit.MILLISECONDS.toMillis(100)
        val deltaTime = currentTimeStamp - lastTimeStamp
        if (deltaTime >= intervalInMilliSeconds) {
            val imageBitmap = image.image?.toBitmap()
            val pixel = imageBitmap!!.getPixel((imageBitmap.width / 2), (imageBitmap.height / 2))
            val red = Color.red(pixel)
            val blue = Color.blue(pixel)
            val green = Color.green(pixel)
            hexColor.onNext(String.format("#%02x%02x%02x", red, green, blue))
            Log.d(TAG, "Color: ${hexColor.value}")
            lastTimeStamp = currentTimeStamp
        }
    }
    // convert the image into a bitmap
    private fun Image.toBitmap(): Bitmap {
        val yBuffer = planes[0].buffer // Y
        val uBuffer = planes[1].buffer // U
        val vBuffer = planes[2].buffer // V
        val ySize = yBuffer.remaining()
        val uSize = uBuffer.remaining()
        val vSize = vBuffer.remaining()
        val nv21 = ByteArray(ySize + uSize + vSize)
        yBuffer.get(nv21, 0, ySize)
        vBuffer.get(nv21, ySize, vSize)
        uBuffer.get(nv21, ySize + vSize, uSize)
        val yuvImage = YuvImage(nv21, ImageFormat.NV21, this.width, this.height, null)
        val out = ByteArrayOutputStream()
        yuvImage.compressToJpeg(Rect(0, 0, yuvImage.width, yuvImage.height), 50, out)
        val imageBytes = out.toByteArray()
        return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
    }
}
**Main Activity:**
/* Get the color from Color Analyzer Class */
private fun createColorAnalyzer(): ImageAnalysis {
    val analyzerConfig = ImageAnalysisConfig.Builder().apply {
        setLensFacing(lensFacing)
        setImageReaderMode(ImageAnalysis.ImageReaderMode.ACQUIRE_LATEST_IMAGE)
    }.build()
    val analyzer = ImageAnalysis(analyzerConfig).apply {
        val colorAnalyzer = ColorAnalyzer()
        colorAnalyzer.hexColor
            .observeOn(AndroidSchedulers.mainThread())
            .subscribe({
                // success
                colorName.text = it.toString() // hexa code in the textView
                colorName.setBackgroundColor(Color.parseColor(it.toString())) // background color of the textView
                (sight.drawable as GradientDrawable).setStroke(10, Color.parseColor(it.toString())) // border color of the sight in the middle of the screen
            }, {
                // error
                Log.d(TAG, "Can not get color :(")
            })
        setAnalyzer(executor, colorAnalyzer)
    }
    return analyzer
}
Hope it will be useful for someone ;)
**EDIT:**
If you read the @Minhaz answer getting the color by doing image -> bitmap -> getPixel() is not very efficient. The most effective is to do image -> RGB.
So here's the Minhaz answer working with Kotlin.
**Color Analyzer Class:**
class ColorAnalyzer : ImageAnalysis.Analyzer {
    private var lastAnalyzedTimestamp = 0L
    private fun ByteBuffer.toByteArray(): ByteArray {
        rewind()    // Rewind the buffer to zero
        val data = ByteArray(remaining())
        get(data)   // Copy the buffer into a byte array
        return data // Return the byte array
    }
    private fun getRGBfromYUV(image: ImageProxy): Triple<Double, Double, Double> {
        val planes = image.planes
        val height = image.height
        val width = image.width
        // Y
        val yArr = planes[0].buffer
        val yArrByteArray = yArr.toByteArray()
        val yPixelStride = planes[0].pixelStride
        val yRowStride = planes[0].rowStride
        // U
        val uArr = planes[1].buffer
        val uArrByteArray = uArr.toByteArray()
        val uPixelStride = planes[1].pixelStride
        val uRowStride = planes[1].rowStride
        // V
        val vArr = planes[2].buffer
        val vArrByteArray = vArr.toByteArray()
        val vPixelStride = planes[2].pixelStride
        val vRowStride = planes[2].rowStride
        val y = yArrByteArray[(height * yRowStride + width * yPixelStride) / 2].toInt() and 255
        val u = (uArrByteArray[(height * uRowStride + width * uPixelStride) / 4].toInt() and 255) - 128
        val v = (vArrByteArray[(height * vRowStride + width * vPixelStride) / 4].toInt() and 255) - 128
        val r = y + (1.370705 * v)
        val g = y - (0.698001 * v) - (0.337633 * u)
        val b = y + (1.732446 * u)
        return Triple(r, g, b)
    }
    // analyze the color
    override fun analyze(image: ImageProxy, rotationDegrees: Int) {
        val currentTimestamp = System.currentTimeMillis()
        if (currentTimestamp - lastAnalyzedTimestamp >= TimeUnit.MILLISECONDS.toMillis(100)) {
            val colors = getRGBfromYUV(image)
            var hexColor = String.format("#%02x%02x%02x", colors.first.toInt(), colors.second.toInt(), colors.third.toInt())
            Log.d("test", "hexColor: $hexColor")
            lastAnalyzedTimestamp = currentTimestamp
        }
    }
}

planes = imageProxy.getPlanes()
val height = imageProxy.getHeight()
val width = imageProxy.getWidth()
// You may have to find the logic to get array from ByteBuffer
// Y
val yArr = planes[0].buffer.array()
val yPixelStride = planes[0].getPixelStride()
val yRowStride = planes[0].getRowStride()
// U
val uArr = planes[1].buffer.array()
val uPixelStride = planes[1].getPixelStride()
val uRowStride = planes[1].getRowStride()
// V
val vArr = planes[2].buffer.array()
val vPixelStride = planes[2].getPixelStride()
val vRowStride = planes[2].getRowStride()
val y = yArr[(height * yRowStride + width * yPixelStride) / 2] & 255 
val u = (uArr[(height * uRowStride + width * uPixelStride) / 4] & 255) - 128
val v = (vArr[(height * vRowStride + width * vPixelStride) / 4] & 255) - 128 
val r = y + (1.370705 * v);
val g = y - (0.698001 * v) - (0.337633 * u);
val b = y + (1.732446 * u);