在使用cgo调用和运行Python3.7的numpy矩阵点乘操作时,无法提取结果。

huangapple go评论94阅读模式
英文:

Can not extract the results when using cgo call and run numpy matrix dot operation of python3.7

问题

我不知道result的类型,因此我不知道如何提取其中的值并将其导出到go数组中。以下是可以编译和运行的代码。

package main

/*
#cgo LDFLAGS: -lm
#cgo pkg-config: python3
#include <Python.h>
*/
import "C"
import (
	"fmt"

	"github.com/DataDog/go-python3"
)

func main() {
	python3.Py_Initialize()
	numpy := python3.PyImport_ImportModule("numpy")
	dotFunc := numpy.GetAttrString("dot")

	array1 := python3.PyList_New(2)
	python3.PyList_SetItem(array1, 0, python3.PyList_New(2))
	python3.PyList_SetItem(array1, 1, python3.PyList_New(2))

	python3.PyList_SetItem(python3.PyList_GetItem(array1, 0), 0, python3.PyFloat_FromDouble(1.0))
	python3.PyList_SetItem(python3.PyList_GetItem(array1, 0), 1, python3.PyFloat_FromDouble(2.0))
	python3.PyList_SetItem(python3.PyList_GetItem(array1, 1), 0, python3.PyFloat_FromDouble(3.0))
	python3.PyList_SetItem(python3.PyList_GetItem(array1, 1), 1, python3.PyFloat_FromDouble(4.0))

	array2 := python3.PyList_New(2)
	python3.PyList_SetItem(array2, 0, python3.PyList_New(2))
	python3.PyList_SetItem(array2, 1, python3.PyList_New(2))

	python3.PyList_SetItem(python3.PyList_GetItem(array2, 0), 0, python3.PyFloat_FromDouble(5.0))
	python3.PyList_SetItem(python3.PyList_GetItem(array2, 0), 1, python3.PyFloat_FromDouble(6.0))
	python3.PyList_SetItem(python3.PyList_GetItem(array2, 1), 0, python3.PyFloat_FromDouble(7.0))
	python3.PyList_SetItem(python3.PyList_GetItem(array2, 1), 1, python3.PyFloat_FromDouble(8.0))

	result := dotFunc.CallFunctionObjArgs(array1, array2)
	fmt.Println(array1)
	fmt.Println(array2)
	fmt.Println(result)

	fmt.Println(python3.PyList_Size(array1))       //返回2
	fmt.Println(python3.PyList_Size(array2))       //返回2
	fmt.Println(python3.PyType_CheckExact(result)) //返回false,不知道它的类型

}

根据第一个答案进行了更新(同时我清理了代码以避免导入go-python3),我应该将numpy数组转换为c数组:

package main

/*
#cgo CFLAGS: -I/anaconda3/include/python3.7m -I/lib/python3.7/site-packages/numpy/core/include
#cgo LDFLAGS: -lm -L/anaconda3/lib -lpython3.7m
#include <Python.h>
#include <numpy/arrayobject.h>
*/
import "C"
import (
	"fmt"
)

func main() {
	// 初始化Python解释器
	C.Py_Initialize()
	numpyModule := C.PyImport_ImportModule(C.CString("numpy"))
	dotFunc := C.PyObject_GetAttrString(numpyModule, C.CString("dot"))

	array1 := C.PyList_New(2)
	C.PyList_SetItem(array1, 0, C.PyList_New(2))
	C.PyList_SetItem(array1, 1, C.PyList_New(2))

	C.PyList_SetItem(C.PyList_GetItem(array1, 0), 0, C.PyFloat_FromDouble(1.0))
	C.PyList_SetItem(C.PyList_GetItem(array1, 0), 1, C.PyFloat_FromDouble(2.0))
	C.PyList_SetItem(C.PyList_GetItem(array1, 1), 0, C.PyFloat_FromDouble(3.0))
	C.PyList_SetItem(C.PyList_GetItem(array1, 1), 1, C.PyFloat_FromDouble(4.0))

	array2 := C.PyList_New(2)
	C.PyList_SetItem(array2, 0, C.PyList_New(2))
	C.PyList_SetItem(array2, 1, C.PyList_New(2))

	C.PyList_SetItem(C.PyList_GetItem(array2, 0), 0, C.PyFloat_FromDouble(5.0))
	C.PyList_SetItem(C.PyList_GetItem(array2, 0), 1, C.PyFloat_FromDouble(6.0))
	C.PyList_SetItem(C.PyList_GetItem(array2, 1), 0, C.PyFloat_FromDouble(7.0))
	C.PyList_SetItem(C.PyList_GetItem(array2, 1), 1, C.PyFloat_FromDouble(8.0))

	args := C.PyTuple_New(2)
	C.PyTuple_SetItem(args, 0, array1)
	C.PyTuple_SetItem(args, 1, array2)

	result := C.PyObject_CallObject(dotFunc, args) // 返回np.array

	result = (*C.double)(C.PyArray_DATA(result)) // np.array转换为c数组

	fmt.Println(result)

	C.Py_Finalize()
}

现在唯一的问题是:go build返回错误could not determine kind of name for C.PyArray_DATA。所以我在CFLAGS、LDFLAGS或其他地方漏掉了什么?

英文:

I dont know the type of resut, thus I dont know how to extact the values in result and export it to the go array. Below is the code, which can be compiled and run.

package main
/*
#cgo LDFLAGS: -lm
#cgo pkg-config: python3
#include &lt;Python.h&gt;
*/
import &quot;C&quot;
import (
&quot;fmt&quot;
&quot;github.com/DataDog/go-python3&quot;
)
func main() {
python3.Py_Initialize()
numpy := python3.PyImport_ImportModule(&quot;numpy&quot;)
dotFunc := numpy.GetAttrString(&quot;dot&quot;)
array1 := python3.PyList_New(2)
python3.PyList_SetItem(array1, 0, python3.PyList_New(2))
python3.PyList_SetItem(array1, 1, python3.PyList_New(2))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 0), 0, python3.PyFloat_FromDouble(1.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 0), 1, python3.PyFloat_FromDouble(2.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 1), 0, python3.PyFloat_FromDouble(3.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 1), 1, python3.PyFloat_FromDouble(4.0))
array2 := python3.PyList_New(2)
python3.PyList_SetItem(array2, 0, python3.PyList_New(2))
python3.PyList_SetItem(array2, 1, python3.PyList_New(2))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 0), 0, python3.PyFloat_FromDouble(5.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 0), 1, python3.PyFloat_FromDouble(6.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 1), 0, python3.PyFloat_FromDouble(7.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 1), 1, python3.PyFloat_FromDouble(8.0))
result := dotFunc.CallFunctionObjArgs(array1, array2)
fmt.Println(array1)
fmt.Println(array2)
fmt.Println(result)
fmt.Println(python3.PyList_Size(array1))       //return 2
fmt.Println(python3.PyList_Size(array2))       //retunr 2
fmt.Println(python3.PyType_CheckExact(result)) //return false, does not know its type
}

Updated based on the first answer (also I clean up the codes to avoid importing go-python3), I should convert numpy array to c array:

package main
/*
#cgo CFLAGS: -I/anaconda3/include/python3.7m -I/lib/python3.7/site-packages/numpy/core/include
#cgo LDFLAGS: -lm -L/anaconda3/lib -lpython3.7m
#include &lt;Python.h&gt;
#include &lt;numpy/arrayobject.h&gt;
*/
import &quot;C&quot;
import (
&quot;fmt&quot;
)
func main() {
// Initialize the Python interpreter
C.Py_Initialize()
numpyModule := C.PyImport_ImportModule(C.CString(&quot;numpy&quot;))
dotFunc := C.PyObject_GetAttrString(numpyModule, C.CString(&quot;dot&quot;))
array1 := C.PyList_New(2)
C.PyList_SetItem(array1, 0, C.PyList_New(2))
C.PyList_SetItem(array1, 1, C.PyList_New(2))
C.PyList_SetItem(C.PyList_GetItem(array1, 0), 0, C.PyFloat_FromDouble(1.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 0), 1, C.PyFloat_FromDouble(2.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 1), 0, C.PyFloat_FromDouble(3.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 1), 1, C.PyFloat_FromDouble(4.0))
array2 := C.PyList_New(2)
C.PyList_SetItem(array2, 0, C.PyList_New(2))
C.PyList_SetItem(array2, 1, C.PyList_New(2))
C.PyList_SetItem(C.PyList_GetItem(array2, 0), 0, C.PyFloat_FromDouble(5.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 0), 1, C.PyFloat_FromDouble(6.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 1), 0, C.PyFloat_FromDouble(7.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 1), 1, C.PyFloat_FromDouble(8.0))
args := C.PyTuple_New(2)
C.PyTuple_SetItem(args, 0, array1)
C.PyTuple_SetItem(args, 1, array2)
result := C.PyObject_CallObject(dotFunc, args) // return np.array
result = (*C.double)(C.PyArray_DATA(result)) // np.array to c array
fmt.Println(result)
C.Py_Finalize()
}

The only problem now is: go build return error could not determine kind of name for C.PyArray_DATA. So I missing something in CFLAGS, LDFLAGS or other place?

答案1

得分: 1

我个人而言,对于你的示例中提供的 Python3 桥接包,我无法提供太多帮助,因为它不支持 Python ≥ 3.7,它的衍生分支也不支持,而且我的 Debian 系统已经安装了 3.11,所以这些包在我的系统上无法构建。尽管如此,我会尝试提供一条可能的前进路径。

如果你在 python3 会话中加载 numpy 并查看 numpy.dot 的文档 -

>>> import numpy
>>> help(numpy.dot)

...你会注意到它的文档中提到返回一个由 numpy 提供的自定义类型的值 - numpy.ndarray,而在 Python 中,所有自定义类型本质上都是类。你可以使用 help(numpy.ndarray) 来了解这个类型的详细信息。

根据这些信息,我们可以使用 Python 的对象(类的实例)处理协议来处理感兴趣的调用返回的值:你可以查询任何对象的类型、它支持的属性、调用它的方法等等,而你正在使用的包实现了必要的支持,你可以在该包的一个名为 "list" 的示例中看到它的实际应用。

以下是一些杂项说明。

  • 你的示例似乎并没有实际使用 cgo,所以我不确定它是否需要 - 除非这是你正在处理的真实代码中的残留部分。

  • 很可能你实际上并不想生成一个 Go 的 array,而是一个切片。如果你不了解这个区别,请从这里开始,并在这里继续,在继续你的工作之前。

  • 正如你从 numpy.ndarray 的文档中所看到的,这个类型是一个相当复杂的东西,因为它表面上支持任意维度的数组。

    看起来前进的方法是创建一个辅助函数,它接受一个 *python3.PyObject,验证它是否具有预期的类型,验证它是否具有预期的维度,然后提取数据 - 可能使用对象协议的 __index__() 方法。

  • 如果你只是想在 Go 中进行矩阵乘法,并且实际上并不需要与一些已有的、大量使用 numpy 的 Python 代码进行接口交互,我建议你使用一个 Go 本地的解决方案,比如 gonum,它在其 package mat 中支持点积。
    该包提供了在 amd64 上使用汇编编写的实现,因此在相似的数据集上,性能应该至少与使用 numpy 的 Python 代码相当,并且最终结果将更容易支持和部署。

英文:

I, for one, cannot be of much help as the package providing the Python3 bridge in your example does not work with Python ≥ 3.7, its blessed fork does not, either, and even my Debian system already has 3.11, so the packages simply do not build on my system. Still, I'll try to offer a possible path forward.

If you load up numpy in a python3 session and see the docs on numpy.dot

&gt;&gt;&gt; import numpy
&gt;&gt;&gt; help(numpy.dot)

…you will notice it's documented to return a value of a custom type, provided by numpy–a numpy.ndarray, and all custom types in Python are inherently classes. You can read up on that type using help(numpy.ndarray).

Given this information, we can simply approach the value returned by the call of interest using the "stock" Python's protocol of working with objects (instances of classes): you can query any object about its type, the attributes it supports, call methods on it and so on, and the package you're using implements the necessary support, which you can see in action in one of the package's examples called "list".

A couple of assorted notes follows.

  • Your example does not appear to actually use cgo, so I'm not sure it's needed–unless that's some remnant from the real code you're dealing with.

  • Quite possibly you don't actually want to produce a Go's array, but rather a slice. If you do not know the distinction, start here, and continue there before continuing with your endaevor.

  • As you should see from the numpy.ndarray documentation, this type is quite a complicated thing as it ostensibly supports arrays of arbitrary dimensions.

    Looks like a way forward is creating a helper function which would take an *python3.PyObject, verify it has the expected type, verify it has the expected dimensions and then extract the data–supposedly using the __index__() method of the object protocol.

  • If you really just want to multiply matrices in Go, and are not actually tasked with interfacing some big existing well-tested Python code which makes heavy use of numpy, I would recommend to instead use a Go-native solution such as gonum which supports dot product in its package mat.
    This package provides implementations written in assembly for amd64, so the performance on comparable datasets should at least be on par with Python code using numpy and the end result will be much easier to support and ship.

huangapple
  • 本文由 发表于 2023年5月18日 18:32:55
  • 转载请务必保留本文链接:https://go.coder-hub.com/76279918.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定