如何使用Pion/Webrtc将VP8帧转换为图像?

huangapple go评论85阅读模式
英文:

How to convert VP8 interframe into image with Pion/Webrtc?

问题

我已经尝试了Pion/Webrtc examples中的快照示例,将VP8视频流转换为JPEG。

它运行良好,但是不包括间隔帧。当我在修复时,显示了一个错误消息。

vp8: Golden / AltRef frames are not implemented.

这个错误来自golang的vp8包

修改后的代码如下所示:

if !videoKeyframe {
	decoder.Init(bytes.NewReader(sample.Data), len(sample.Data))

	_, err := decoder.DecodeFrameHeader()
	if err != nil {
		fmt.Printf("Header Error: %s\n", err)
		continue
	}

	_, err2 := decoder.DecodeFrame()
	if err2 != nil {
		fmt.Printf("DecodeFrame Error: %s\n", err2)
		continue
	}
}

我还尝试修改了gstreamer-receive示例,但是用C语言很难实现。

有没有简单地将间隔帧转换为JPEG的方法?

英文:

I have tried snapshot example to convert VP8 video stream to jpeg in Pion/Webrtc examples.

It works well, but interframes are not included. An error message displayed when I was fixing it.

vp8: Golden / AltRef frames are not implemented.

This error comes from golang's vp8 package.

The modified code is shown below:

if !videoKeyframe {
	decoder.Init(bytes.NewReader(sample.Data), len(sample.Data))

	_, err := decoder.DecodeFrameHeader()
	if err != nil {
		fmt.Printf("Header Error: %s\n", err)
		continue
	}

	_, err2 := decoder.DecodeFrame()
	if err2 != nil {
		fmt.Printf("DecodeFrame Error: %s\n", err2)
		continue
	}
} 

I also tried to modify gstreamer-receive example. But it's hard for me to do in C.

Is there anyway to convert interframe to jpeg easily ?

答案1

得分: 3

最后,我使用libvpx-go解决了问题。

我修改的代码如下所示:

package main

import (
	"bytes"
	"fmt"
	"image/jpeg"

	//	"log"
	"os"

	"github.com/pion/rtp"
	"github.com/pion/rtp/codecs"
	"github.com/pion/webrtc/v3/pkg/media/samplebuilder"
	"github.com/xlab/libvpx-go/vpx"
)

type VDecoder struct {
	enabled bool

	src   <-chan *rtp.Packet
	ctx   *vpx.CodecCtx
	iface *vpx.CodecIface
}

type VCodec string

const (
	CodecVP8  VCodec = "V_VP8"
	CodecVP9  VCodec = "V_VP9"
	CodecVP10 VCodec = "V_VP10"
)

func NewVDecoder(codec VCodec, src <-chan *rtp.Packet) *VDecoder {
	dec := &VDecoder{
		src: src,
		ctx: vpx.NewCodecCtx(),
	}
	switch codec {
	case CodecVP8:
		dec.iface = vpx.DecoderIfaceVP8()
	case CodecVP9:
		dec.iface = vpx.DecoderIfaceVP9()
	default: // others are currently disabled
		log.Println("[WARN] unsupported VPX codec:", codec)
		return dec
	}
	err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
	if err != nil {
		log.Println("[WARN]", err)
		return dec
	}
	dec.enabled = true
	return dec
}

func (v *VDecoder) Save(savePath string) { //, out chan<- Frame
	//	defer close(out)
	i := 0

	sampleBuilder := samplebuilder.New(20000, &codecs.VP8Packet{}, 90000)

	for pkt := range v.src {

		sampleBuilder.Push(pkt)

		// Use SampleBuilder to generate full picture from many RTP Packets
		sample := sampleBuilder.Pop()
		if sample == nil {
			continue
		}

		if !v.enabled {
			continue
		}

		dataSize := uint32(len(sample.Data))

		err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample.Data), dataSize, nil, 0))
		if err != nil {
			log.Println("[WARN]", err)
			continue
		}

		var iter vpx.CodecIter
		img := vpx.CodecGetFrame(v.ctx, &iter)
		if img != nil {
			img.Deref()

			// out <- Frame{
			// 	RGBA:     img.ImageRGBA(),
			// 	Timecode: time.Duration(pkt.Timestamp),
			// }

			i++

			buffer := new(bytes.Buffer)
			if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
				//	panic(err)
				fmt.Printf("jpeg Encode Error: %s\r\n", err)
			}

			fo, err := os.Create(fmt.Sprintf("%s%d%s", savePath, i, ".jpg"))

			if err != nil {
				fmt.Printf("image create Error: %s\r\n", err)
				//panic(err)
			}
			// close fo on exit and check for its returned error
			defer func() {
				if err := fo.Close(); err != nil {
					panic(err)
				}
			}()

			if _, err := fo.Write(buffer.Bytes()); err != nil {
				fmt.Printf("image write Error: %s\r\n", err)
				//panic(err)
			}

			fo.Close()
		}
	}
}

只需添加两行代码以保存连续的图像:

vd := NewVDecoder(CodecVP8, rtpChan)

go vd.Save("/ramdisk/image_seq/xxx")

享受吧!

P.S. 在运行go get之前,不要忘记安装libvpx-go的依赖项,并确保libvpx的版本为1.8

英文:

Finally, I solved with libvpx-go

I modified code as shown below.

package main
import (
&quot;bytes&quot;
&quot;fmt&quot;
&quot;image/jpeg&quot;
//	&quot;log&quot;
&quot;os&quot;
&quot;github.com/pion/rtp&quot;
&quot;github.com/pion/rtp/codecs&quot;
&quot;github.com/pion/webrtc/v3/pkg/media/samplebuilder&quot;
&quot;github.com/xlab/libvpx-go/vpx&quot;
)
// type Frame struct {
// 	*image.RGBA
// 	Timecode   time.Duration
// 	IsKeyframe bool
// }
type VDecoder struct {
enabled bool
src   &lt;-chan *rtp.Packet
ctx   *vpx.CodecCtx
iface *vpx.CodecIface
}
type VCodec string
const (
CodecVP8  VCodec = &quot;V_VP8&quot;
CodecVP9  VCodec = &quot;V_VP9&quot;
CodecVP10 VCodec = &quot;V_VP10&quot;
)
func NewVDecoder(codec VCodec, src &lt;-chan *rtp.Packet) *VDecoder {
dec := &amp;VDecoder{
src: src,
ctx: vpx.NewCodecCtx(),
}
switch codec {
case CodecVP8:
dec.iface = vpx.DecoderIfaceVP8()
case CodecVP9:
dec.iface = vpx.DecoderIfaceVP9()
default: // others are currently disabled
log.Println(&quot;[WARN] unsupported VPX codec:&quot;, codec)
return dec
}
err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
if err != nil {
log.Println(&quot;[WARN]&quot;, err)
return dec
}
dec.enabled = true
return dec
}
func (v *VDecoder) Save(savePath string) { //, out chan&lt;- Frame
//	defer close(out)
i := 0
sampleBuilder := samplebuilder.New(20000, &amp;codecs.VP8Packet{}, 90000)
for pkt := range v.src {
sampleBuilder.Push(pkt)
// Use SampleBuilder to generate full picture from many RTP Packets
sample := sampleBuilder.Pop()
if sample == nil {
continue
}
if !v.enabled {
continue
}
dataSize := uint32(len(sample.Data))
err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample.Data), dataSize, nil, 0))
if err != nil {
log.Println(&quot;[WARN]&quot;, err)
continue
}
var iter vpx.CodecIter
img := vpx.CodecGetFrame(v.ctx, &amp;iter)
if img != nil {
img.Deref()
// out &lt;- Frame{
// 	RGBA:     img.ImageRGBA(),
// 	Timecode: time.Duration(pkt.Timestamp),
// }
i++
buffer := new(bytes.Buffer)
if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
//	panic(err)
fmt.Printf(&quot;jpeg Encode Error: %s\r\n&quot;, err)
}
fo, err := os.Create(fmt.Sprintf(&quot;%s%d%s&quot;, savePath, i, &quot;.jpg&quot;))
if err != nil {
fmt.Printf(&quot;image create Error: %s\r\n&quot;, err)
//panic(err)
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
if _, err := fo.Write(buffer.Bytes()); err != nil {
fmt.Printf(&quot;image write Error: %s\r\n&quot;, err)
//panic(err)
}
fo.Close()
}
}
}

Just add 2 lines of code to save sequential images

vd := NewVDecoder(CodecVP8, rtpChan)
go vd.Save(&quot;/ramdisk/image_seq/xxx&quot;)

Enjoy!

P.S. don't forget to install libvpx-go 's dependencies before running go get and make sure libvpx 's version is 1.8

答案2

得分: 1

如果您需要将帧间编码转换为JPEG格式,唯一的方法是解码流中的每一帧。我认为没有其他方法,因为帧间编码依赖于前一帧。

英文:

If you need to convert interframe to jpeg the only way would be to decode every frame from your stream. I don't think there is any other way as interframes depend on previous frames.

答案3

得分: 0

同步解决方案

这是与CK So相同解决方案的另一个版本,它基于libvpx-go,但不使用通道。

package main

import (
	"bytes"
	"fmt"
	"image/jpeg"
	"log"
	"os"

	"github.com/xlab/libvpx-go/vpx"
)

type VDecoderSync struct {
	enabled bool
	ctx     *vpx.CodecCtx
	iface   *vpx.CodecIface
	i       int
}

type VCodecSync string

const (
	CodecVP8Sync  VCodecSync = "V_VP8"
	CodecVP9Sync  VCodecSync = "V_VP9"
	CodecVP10Sync VCodecSync = "V_VP10"
)

func NewVDecoderSync(codec VCodecSync) *VDecoderSync {
	dec := &VDecoderSync{
		ctx: vpx.NewCodecCtx(),
	}
	switch codec {
	case CodecVP8Sync:
		dec.iface = vpx.DecoderIfaceVP8()
	case CodecVP9Sync:
		dec.iface = vpx.DecoderIfaceVP9()
	default:
		log.Println("[WARN] unsupported VPX codec:", codec)
		return dec
	}
	err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
	if err != nil {
		log.Println("[WARN]", err)
		return dec
	}
	dec.enabled = true
	return dec
}

func (v *VDecoderSync) EncodeToJson(savePath string, pkt []byte, print_to_file bool) []byte {
	sample := pkt
	dataSize := uint32(len(sample))
	err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample), dataSize, nil, 0))
	if err != nil {
		log.Println("[WARN]", err)
		return nil
	}
	var iter vpx.CodecIter
	img := vpx.CodecGetFrame(v.ctx, &iter)
	if img != nil {
		img.Deref()
		buffer := new(bytes.Buffer)
		if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
			fmt.Printf("jpeg Encode Error: %s\r\n", err)
		}
		bytes_to_send := buffer.Bytes()
		if print_to_file {
			v.i++
			v.i = v.i % 30
			fo, err := os.Create(fmt.Sprintf("%s%d%s", savePath, v.i, ".jpg"))
			if err != nil {
				fmt.Printf("image create Error: %s\r\n", err)
			}
			defer func() {
				if err := fo.Close(); err != nil {
					panic(err)
				}
			}()
			if _, err := fo.Write(bytes_to_send); err != nil {
				fmt.Printf("image write Error: %s\r\n", err)
			}
		}
		return bytes_to_send
	}
	return nil
}

希望这可以帮助到你!如果你有任何其他问题,请随时问。

英文:

Sync Solution

This is an alternative version of the same solution of CK So
It is based on libvpx-go but dose not use channels.

package main
import (
&quot;bytes&quot;
&quot;fmt&quot;
&quot;image/jpeg&quot;
&quot;log&quot;
//  &quot;log&quot;
&quot;os&quot;
&quot;github.com/xlab/libvpx-go/vpx&quot;
)
// type Frame struct {
//  *image.RGBA
//  Timecode   time.Duration
//  IsKeyframe bool
// }
type VDecoderSync struct {
enabled bool
//src   &lt;-chan []byte this blocked the thread even if fed.
ctx   *vpx.CodecCtx
iface *vpx.CodecIface
i     int
}
type VCodecSync string
const (
CodecVP8Sync  VCodecSync = &quot;V_VP8&quot;
CodecVP9Sync  VCodecSync = &quot;V_VP9&quot;
CodecVP10Sync VCodecSync = &quot;V_VP10&quot;
)
func NewVDecoderSync(codec VCodecSync) *VDecoderSync {
dec := &amp;VDecoderSync{
ctx: vpx.NewCodecCtx(),
}
switch codec {
case CodecVP8Sync:
dec.iface = vpx.DecoderIfaceVP8()
case CodecVP9Sync:
dec.iface = vpx.DecoderIfaceVP9()
default: // others are currently disabled
log.Println(&quot;[WARN] unsupported VPX codec:&quot;, codec)
return dec
}
err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
if err != nil {
log.Println(&quot;[WARN]&quot;, err)
return dec
}
dec.enabled = true
return dec
}
func (v *VDecoderSync) EncodeToJson(savePath string, pkt []byte, print_to_file bool) []byte {
sample := pkt
dataSize := uint32(len(sample))
err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample), dataSize, nil, 0))
if err != nil {
log.Println(&quot;[WARN]&quot;, err)
return nil
}
var iter vpx.CodecIter
img := vpx.CodecGetFrame(v.ctx, &amp;iter)
if img != nil {
img.Deref()
// out &lt;- Frame{
//  RGBA:     img.ImageRGBA(),
//  Timecode: time.Duration(pkt.Timestamp),
// }
buffer := new(bytes.Buffer)
if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
//  panic(err)
fmt.Printf(&quot;jpeg Encode Error: %s\r\n&quot;, err)
}
bytes_to_send := buffer.Bytes()
if print_to_file {
v.i++
v.i = v.i % 30
fo, err := os.Create(fmt.Sprintf(&quot;%s%d%s&quot;, savePath, v.i, &quot;.jpg&quot;))
if err != nil {
fmt.Printf(&quot;image create Error: %s\r\n&quot;, err)
//panic(err)
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
if _, err := fo.Write(bytes_to_send); err != nil {
fmt.Printf(&quot;image write Error: %s\r\n&quot;, err)
//panic(err)
}
}
return bytes_to_send
}
return nil
}

huangapple
  • 本文由 发表于 2021年8月20日 16:24:34
  • 转载请务必保留本文链接:https://go.coder-hub.com/68859120.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定