How to convert VP8 interframe into image with Pion/Webrtc?

I have tried snapshot example to convert VP8 video stream to jpeg in Pion/Webrtc examples.

It works well, but interframes are not included. An error message displayed when I was fixing it.

vp8: Golden / AltRef frames are not implemented.

This error comes from golang's vp8 package.

The modified code is shown below:

if !videoKeyframe {
	decoder.Init(bytes.NewReader(sample.Data), len(sample.Data))

	_, err := decoder.DecodeFrameHeader()
	if err != nil {
		fmt.Printf("Header Error: %s\n", err)
		continue
	}

	_, err2 := decoder.DecodeFrame()
	if err2 != nil {
		fmt.Printf("DecodeFrame Error: %s\n", err2)
		continue
	}
} 

I also tried to modify gstreamer-receive example. But it's hard for me to do in C.

Is there anyway to convert interframe to jpeg easily ?

Finally, I solved with libvpx-go

I modified code as shown below.

package main
import (
"bytes"
"fmt"
"image/jpeg"
//	"log"
"os"
"github.com/pion/rtp"
"github.com/pion/rtp/codecs"
"github.com/pion/webrtc/v3/pkg/media/samplebuilder"
"github.com/xlab/libvpx-go/vpx"
)
// type Frame struct {
// 	*image.RGBA
// 	Timecode   time.Duration
// 	IsKeyframe bool
// }
type VDecoder struct {
enabled bool
src   <-chan *rtp.Packet
ctx   *vpx.CodecCtx
iface *vpx.CodecIface
}
type VCodec string
const (
CodecVP8  VCodec = "V_VP8"
CodecVP9  VCodec = "V_VP9"
CodecVP10 VCodec = "V_VP10"
)
func NewVDecoder(codec VCodec, src <-chan *rtp.Packet) *VDecoder {
dec := &VDecoder{
src: src,
ctx: vpx.NewCodecCtx(),
}
switch codec {
case CodecVP8:
dec.iface = vpx.DecoderIfaceVP8()
case CodecVP9:
dec.iface = vpx.DecoderIfaceVP9()
default: // others are currently disabled
log.Println("[WARN] unsupported VPX codec:", codec)
return dec
}
err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
if err != nil {
log.Println("[WARN]", err)
return dec
}
dec.enabled = true
return dec
}
func (v *VDecoder) Save(savePath string) { //, out chan<- Frame
//	defer close(out)
i := 0
sampleBuilder := samplebuilder.New(20000, &codecs.VP8Packet{}, 90000)
for pkt := range v.src {
sampleBuilder.Push(pkt)
// Use SampleBuilder to generate full picture from many RTP Packets
sample := sampleBuilder.Pop()
if sample == nil {
continue
}
if !v.enabled {
continue
}
dataSize := uint32(len(sample.Data))
err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample.Data), dataSize, nil, 0))
if err != nil {
log.Println("[WARN]", err)
continue
}
var iter vpx.CodecIter
img := vpx.CodecGetFrame(v.ctx, &iter)
if img != nil {
img.Deref()
// out <- Frame{
// 	RGBA:     img.ImageRGBA(),
// 	Timecode: time.Duration(pkt.Timestamp),
// }
i++
buffer := new(bytes.Buffer)
if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
//	panic(err)
fmt.Printf("jpeg Encode Error: %s\r\n", err)
}
fo, err := os.Create(fmt.Sprintf("%s%d%s", savePath, i, ".jpg"))
if err != nil {
fmt.Printf("image create Error: %s\r\n", err)
//panic(err)
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
if _, err := fo.Write(buffer.Bytes()); err != nil {
fmt.Printf("image write Error: %s\r\n", err)
//panic(err)
}
fo.Close()
}
}
}

Just add 2 lines of code to save sequential images

vd := NewVDecoder(CodecVP8, rtpChan)
go vd.Save("/ramdisk/image_seq/xxx")

Enjoy!

P.S. don't forget to install libvpx-go 's dependencies before running go get and make sure libvpx 's version is 1.8

If you need to convert interframe to jpeg the only way would be to decode every frame from your stream. I don't think there is any other way as interframes depend on previous frames.

Sync Solution

This is an alternative version of the same solution of CK So
It is based on libvpx-go but dose not use channels.

package main
import (
"bytes"
"fmt"
"image/jpeg"
"log"
//  "log"
"os"
"github.com/xlab/libvpx-go/vpx"
)
// type Frame struct {
//  *image.RGBA
//  Timecode   time.Duration
//  IsKeyframe bool
// }
type VDecoderSync struct {
enabled bool
//src   <-chan []byte this blocked the thread even if fed.
ctx   *vpx.CodecCtx
iface *vpx.CodecIface
i     int
}
type VCodecSync string
const (
CodecVP8Sync  VCodecSync = "V_VP8"
CodecVP9Sync  VCodecSync = "V_VP9"
CodecVP10Sync VCodecSync = "V_VP10"
)
func NewVDecoderSync(codec VCodecSync) *VDecoderSync {
dec := &VDecoderSync{
ctx: vpx.NewCodecCtx(),
}
switch codec {
case CodecVP8Sync:
dec.iface = vpx.DecoderIfaceVP8()
case CodecVP9Sync:
dec.iface = vpx.DecoderIfaceVP9()
default: // others are currently disabled
log.Println("[WARN] unsupported VPX codec:", codec)
return dec
}
err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
if err != nil {
log.Println("[WARN]", err)
return dec
}
dec.enabled = true
return dec
}
func (v *VDecoderSync) EncodeToJson(savePath string, pkt []byte, print_to_file bool) []byte {
sample := pkt
dataSize := uint32(len(sample))
err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample), dataSize, nil, 0))
if err != nil {
log.Println("[WARN]", err)
return nil
}
var iter vpx.CodecIter
img := vpx.CodecGetFrame(v.ctx, &iter)
if img != nil {
img.Deref()
// out <- Frame{
//  RGBA:     img.ImageRGBA(),
//  Timecode: time.Duration(pkt.Timestamp),
// }
buffer := new(bytes.Buffer)
if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
//  panic(err)
fmt.Printf("jpeg Encode Error: %s\r\n", err)
}
bytes_to_send := buffer.Bytes()
if print_to_file {
v.i++
v.i = v.i % 30
fo, err := os.Create(fmt.Sprintf("%s%d%s", savePath, v.i, ".jpg"))
if err != nil {
fmt.Printf("image create Error: %s\r\n", err)
//panic(err)
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
if _, err := fo.Write(bytes_to_send); err != nil {
fmt.Printf("image write Error: %s\r\n", err)
//panic(err)
}
}
return bytes_to_send
}
return nil
}