英文:
Can not extract the results when using cgo call and run numpy matrix dot operation of python3.7
问题
我不知道result
的类型,因此我不知道如何提取其中的值并将其导出到go数组中。以下是可以编译和运行的代码。
package main
/*
#cgo LDFLAGS: -lm
#cgo pkg-config: python3
#include <Python.h>
*/
import "C"
import (
"fmt"
"github.com/DataDog/go-python3"
)
func main() {
python3.Py_Initialize()
numpy := python3.PyImport_ImportModule("numpy")
dotFunc := numpy.GetAttrString("dot")
array1 := python3.PyList_New(2)
python3.PyList_SetItem(array1, 0, python3.PyList_New(2))
python3.PyList_SetItem(array1, 1, python3.PyList_New(2))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 0), 0, python3.PyFloat_FromDouble(1.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 0), 1, python3.PyFloat_FromDouble(2.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 1), 0, python3.PyFloat_FromDouble(3.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 1), 1, python3.PyFloat_FromDouble(4.0))
array2 := python3.PyList_New(2)
python3.PyList_SetItem(array2, 0, python3.PyList_New(2))
python3.PyList_SetItem(array2, 1, python3.PyList_New(2))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 0), 0, python3.PyFloat_FromDouble(5.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 0), 1, python3.PyFloat_FromDouble(6.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 1), 0, python3.PyFloat_FromDouble(7.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 1), 1, python3.PyFloat_FromDouble(8.0))
result := dotFunc.CallFunctionObjArgs(array1, array2)
fmt.Println(array1)
fmt.Println(array2)
fmt.Println(result)
fmt.Println(python3.PyList_Size(array1)) //返回2
fmt.Println(python3.PyList_Size(array2)) //返回2
fmt.Println(python3.PyType_CheckExact(result)) //返回false,不知道它的类型
}
根据第一个答案进行了更新(同时我清理了代码以避免导入go-python3),我应该将numpy数组转换为c数组:
package main
/*
#cgo CFLAGS: -I/anaconda3/include/python3.7m -I/lib/python3.7/site-packages/numpy/core/include
#cgo LDFLAGS: -lm -L/anaconda3/lib -lpython3.7m
#include <Python.h>
#include <numpy/arrayobject.h>
*/
import "C"
import (
"fmt"
)
func main() {
// 初始化Python解释器
C.Py_Initialize()
numpyModule := C.PyImport_ImportModule(C.CString("numpy"))
dotFunc := C.PyObject_GetAttrString(numpyModule, C.CString("dot"))
array1 := C.PyList_New(2)
C.PyList_SetItem(array1, 0, C.PyList_New(2))
C.PyList_SetItem(array1, 1, C.PyList_New(2))
C.PyList_SetItem(C.PyList_GetItem(array1, 0), 0, C.PyFloat_FromDouble(1.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 0), 1, C.PyFloat_FromDouble(2.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 1), 0, C.PyFloat_FromDouble(3.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 1), 1, C.PyFloat_FromDouble(4.0))
array2 := C.PyList_New(2)
C.PyList_SetItem(array2, 0, C.PyList_New(2))
C.PyList_SetItem(array2, 1, C.PyList_New(2))
C.PyList_SetItem(C.PyList_GetItem(array2, 0), 0, C.PyFloat_FromDouble(5.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 0), 1, C.PyFloat_FromDouble(6.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 1), 0, C.PyFloat_FromDouble(7.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 1), 1, C.PyFloat_FromDouble(8.0))
args := C.PyTuple_New(2)
C.PyTuple_SetItem(args, 0, array1)
C.PyTuple_SetItem(args, 1, array2)
result := C.PyObject_CallObject(dotFunc, args) // 返回np.array
result = (*C.double)(C.PyArray_DATA(result)) // np.array转换为c数组
fmt.Println(result)
C.Py_Finalize()
}
现在唯一的问题是:go build
返回错误could not determine kind of name for C.PyArray_DATA
。所以我在CFLAGS、LDFLAGS或其他地方漏掉了什么?
英文:
I dont know the type of resut, thus I dont know how to extact the values in result
and export it to the go array. Below is the code, which can be compiled and run.
package main
/*
#cgo LDFLAGS: -lm
#cgo pkg-config: python3
#include <Python.h>
*/
import "C"
import (
"fmt"
"github.com/DataDog/go-python3"
)
func main() {
python3.Py_Initialize()
numpy := python3.PyImport_ImportModule("numpy")
dotFunc := numpy.GetAttrString("dot")
array1 := python3.PyList_New(2)
python3.PyList_SetItem(array1, 0, python3.PyList_New(2))
python3.PyList_SetItem(array1, 1, python3.PyList_New(2))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 0), 0, python3.PyFloat_FromDouble(1.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 0), 1, python3.PyFloat_FromDouble(2.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 1), 0, python3.PyFloat_FromDouble(3.0))
python3.PyList_SetItem(python3.PyList_GetItem(array1, 1), 1, python3.PyFloat_FromDouble(4.0))
array2 := python3.PyList_New(2)
python3.PyList_SetItem(array2, 0, python3.PyList_New(2))
python3.PyList_SetItem(array2, 1, python3.PyList_New(2))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 0), 0, python3.PyFloat_FromDouble(5.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 0), 1, python3.PyFloat_FromDouble(6.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 1), 0, python3.PyFloat_FromDouble(7.0))
python3.PyList_SetItem(python3.PyList_GetItem(array2, 1), 1, python3.PyFloat_FromDouble(8.0))
result := dotFunc.CallFunctionObjArgs(array1, array2)
fmt.Println(array1)
fmt.Println(array2)
fmt.Println(result)
fmt.Println(python3.PyList_Size(array1)) //return 2
fmt.Println(python3.PyList_Size(array2)) //retunr 2
fmt.Println(python3.PyType_CheckExact(result)) //return false, does not know its type
}
Updated based on the first answer (also I clean up the codes to avoid importing go-python3), I should convert numpy array to c array:
package main
/*
#cgo CFLAGS: -I/anaconda3/include/python3.7m -I/lib/python3.7/site-packages/numpy/core/include
#cgo LDFLAGS: -lm -L/anaconda3/lib -lpython3.7m
#include <Python.h>
#include <numpy/arrayobject.h>
*/
import "C"
import (
"fmt"
)
func main() {
// Initialize the Python interpreter
C.Py_Initialize()
numpyModule := C.PyImport_ImportModule(C.CString("numpy"))
dotFunc := C.PyObject_GetAttrString(numpyModule, C.CString("dot"))
array1 := C.PyList_New(2)
C.PyList_SetItem(array1, 0, C.PyList_New(2))
C.PyList_SetItem(array1, 1, C.PyList_New(2))
C.PyList_SetItem(C.PyList_GetItem(array1, 0), 0, C.PyFloat_FromDouble(1.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 0), 1, C.PyFloat_FromDouble(2.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 1), 0, C.PyFloat_FromDouble(3.0))
C.PyList_SetItem(C.PyList_GetItem(array1, 1), 1, C.PyFloat_FromDouble(4.0))
array2 := C.PyList_New(2)
C.PyList_SetItem(array2, 0, C.PyList_New(2))
C.PyList_SetItem(array2, 1, C.PyList_New(2))
C.PyList_SetItem(C.PyList_GetItem(array2, 0), 0, C.PyFloat_FromDouble(5.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 0), 1, C.PyFloat_FromDouble(6.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 1), 0, C.PyFloat_FromDouble(7.0))
C.PyList_SetItem(C.PyList_GetItem(array2, 1), 1, C.PyFloat_FromDouble(8.0))
args := C.PyTuple_New(2)
C.PyTuple_SetItem(args, 0, array1)
C.PyTuple_SetItem(args, 1, array2)
result := C.PyObject_CallObject(dotFunc, args) // return np.array
result = (*C.double)(C.PyArray_DATA(result)) // np.array to c array
fmt.Println(result)
C.Py_Finalize()
}
The only problem now is: go build
return error could not determine kind of name for C.PyArray_DATA
. So I missing something in CFLAGS, LDFLAGS or other place?
答案1
得分: 1
我个人而言,对于你的示例中提供的 Python3 桥接包,我无法提供太多帮助,因为它不支持 Python ≥ 3.7,它的衍生分支也不支持,而且我的 Debian 系统已经安装了 3.11,所以这些包在我的系统上无法构建。尽管如此,我会尝试提供一条可能的前进路径。
如果你在 python3
会话中加载 numpy
并查看 numpy.dot
的文档 -
>>> import numpy
>>> help(numpy.dot)
...你会注意到它的文档中提到返回一个由 numpy
提供的自定义类型的值 - numpy.ndarray
,而在 Python 中,所有自定义类型本质上都是类。你可以使用 help(numpy.ndarray)
来了解这个类型的详细信息。
根据这些信息,我们可以使用 Python 的对象(类的实例)处理协议来处理感兴趣的调用返回的值:你可以查询任何对象的类型、它支持的属性、调用它的方法等等,而你正在使用的包实现了必要的支持,你可以在该包的一个名为 "list" 的示例中看到它的实际应用。
以下是一些杂项说明。
-
你的示例似乎并没有实际使用
cgo
,所以我不确定它是否需要 - 除非这是你正在处理的真实代码中的残留部分。 -
很可能你实际上并不想生成一个 Go 的 array,而是一个切片。如果你不了解这个区别,请从这里开始,并在这里继续,在继续你的工作之前。
-
正如你从
numpy.ndarray
的文档中所看到的,这个类型是一个相当复杂的东西,因为它表面上支持任意维度的数组。看起来前进的方法是创建一个辅助函数,它接受一个
*python3.PyObject
,验证它是否具有预期的类型,验证它是否具有预期的维度,然后提取数据 - 可能使用对象协议的__index__()
方法。 -
如果你只是想在 Go 中进行矩阵乘法,并且实际上并不需要与一些已有的、大量使用
numpy
的 Python 代码进行接口交互,我建议你使用一个 Go 本地的解决方案,比如gonum
,它在其 packagemat
中支持点积。
该包提供了在amd64
上使用汇编编写的实现,因此在相似的数据集上,性能应该至少与使用numpy
的 Python 代码相当,并且最终结果将更容易支持和部署。
英文:
I, for one, cannot be of much help as the package providing the Python3 bridge in your example does not work with Python ≥ 3.7, its blessed fork does not, either, and even my Debian system already has 3.11, so the packages simply do not build on my system. Still, I'll try to offer a possible path forward.
If you load up numpy
in a python3
session and see the docs on numpy.dot
–
>>> import numpy
>>> help(numpy.dot)
…you will notice it's documented to return a value of a custom type, provided by numpy
–a numpy.ndarray
, and all custom types in Python are inherently classes. You can read up on that type using help(numpy.ndarray)
.
Given this information, we can simply approach the value returned by the call of interest using the "stock" Python's protocol of working with objects (instances of classes): you can query any object about its type, the attributes it supports, call methods on it and so on, and the package you're using implements the necessary support, which you can see in action in one of the package's examples called "list".
A couple of assorted notes follows.
-
Your example does not appear to actually use
cgo
, so I'm not sure it's needed–unless that's some remnant from the real code you're dealing with. -
Quite possibly you don't actually want to produce a Go's array, but rather a slice. If you do not know the distinction, start here, and continue there before continuing with your endaevor.
-
As you should see from the
numpy.ndarray
documentation, this type is quite a complicated thing as it ostensibly supports arrays of arbitrary dimensions.Looks like a way forward is creating a helper function which would take an
*python3.PyObject
, verify it has the expected type, verify it has the expected dimensions and then extract the data–supposedly using the__index__()
method of the object protocol. -
If you really just want to multiply matrices in Go, and are not actually tasked with interfacing some big existing well-tested Python code which makes heavy use of
numpy
, I would recommend to instead use a Go-native solution such asgonum
which supports dot product in its packagemat
.
This package provides implementations written in assembly foramd64
, so the performance on comparable datasets should at least be on par with Python code usingnumpy
and the end result will be much easier to support and ship.
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论