英文:
unable to convert json to dataframe, polars panicked
问题
Here's the translated code portion:
use reqwest::blocking::get;
use polars::prelude::*;
use serde::{Deserialize, Serialize};
use std::io::Cursor;
#[derive(Serialize, Deserialize, Debug)]
struct StockZhAHist {
date: String,
open: f64,
close: f64,
high: f64,
low: f64,
volume: f64,
turnover: f64,
amplitude: f64,
change_rate: f64,
change_amount: f64,
turnover_rate: f64,
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let url = "http://127.0.0.1:8080/api/public/stock_zh_a_hist";
let response = get(url)?.text()?;
// Deserialize response with serde
let stock_zh_a_hist: Vec<StockZhAHist> = serde_json::from_str(&response)?;
// Convert to polars dataframe
println!("{:?}", stock_zh_a_hist);
let df = JsonReader::new(Cursor::new(response))
.infer_schema_len(Some(1000))
.finish()?;
println!("{:?}", df);
Ok(())
}
Regarding the error you're encountering with Polars, it seems like the Arrow datatype Struct is not supported by Polars. You may need to activate a specific data-type feature to resolve this issue. You mentioned adding the dtype-decimal feature, but it seems it didn't help. You might need to investigate further or seek assistance from the Polars community to determine the correct feature to activate or if there's an alternative approach to handle this specific data type in Polars.
英文:
use reqwest::blocking::get;
use polars::prelude::*;
use serde::{Deserialize, Serialize};
use std::io::Cursor;
#[derive(Serialize, Deserialize, Debug)]
struct StockZhAHist {
date: String,
open: f64,
close: f64,
high: f64,
low: f64,
volume: f64,
turnover: f64,
amplitude: f64,
change_rate: f64,
change_amount: f64,
turnover_rate: f64,
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let url = "http://127.0.0.1:8080/api/public/stock_zh_a_hist";
let response = get(url)?.text()?;
// println!("{}", response);
// deserialize response with serde
let stock_zh_a_hist: Vec<StockZhAHist> = serde_json::from_str(&response)?;
// convert to polars dataframe
println!("{:?}", stock_zh_a_hist);
let df = JsonReader::new(Cursor::new(response))
.infer_schema_len(Some(1000))
.finish()?;
println!("{:?}", df);
Ok(())
}
i have written a script that retrieves raw data from my server and i was able to print out the JSON 'stock_zh_a_hist'. but somehow when i tried to print out the dataframe converted from this JSON, the programme panicked and returned :
> thread 'main' panicked at 'Arrow datatype Struct([Field { name:
> "date", data_type: LargeUtf8, is_nullable: true, metadata: {} }, Field
> { name: "open", data_type: Float64, is_nullable: true, metadata: {} },
> Field { name: "close", data_type: Float64, is_nullable: true,
> metadata: {} }, Field { name: "high", data_type: Float64, is_nullable:
> true, metadata: {} }, Field { name: "low", data_type: Float64,
> is_nullable: true, metadata: {} }, Field { name: "volume", data_type:
> Int64, is_nullable: true, metadata: {} }, Field { name: "turnover",
> data_type: Float64, is_nullable: true, metadata: {} }, Field { name:
> "amplitude", data_type: Float64, is_nullable: true, metadata: {} },
> Field { name: "change_rate", data_type: Float64, is_nullable: true,
> metadata: {} }, Field { name: "change_amount", data_type: Float64,
> is_nullable: true, metadata: {} }, Field { name: "turnover_rate",
> data_type: Float64, is_nullable: true, metadata: {} }]) not supported
> by Polars. You probably need to activate that data-type feature.',
> /home/arthur/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-core-0.30.0/src/datatypes/field.rs:158:19.
i also read source code there which reads:
#[cfg(feature = "dtype-decimal")]
ArrowDataType::Decimal(precision, scale) => DataType::Decimal(Some(*precision), Some(*scale)),
dt => panic!("Arrow datatype {dt:?} not supported by Polars. You probably need to activate that data-type feature."),
i am assuming polars is having some problem reading the json with correct datatypes, but how can i solve it?
update 1:
i have added the feature dtype-decimal in cargo.toml but the same error persists.
update 2:
the content of stock_zh_a_hist is as follows:
[StockZhAHist { date: "1996-12-16T00:00:00.000", open: 16.86, close: 16.86, high: 16.86, low: 16.86, volume: 62442.0, turnover: 105277000.0, amplitude: 0.0, change_rate: -10.22, change_amount: -1.92, turnover_rate: 0.87 },
StockZhAHist { date: "1996-12-17T00:00:00.000", open: 15.17, close: 15.17, high: 16.79, low: 15.17, volume: 463675.0, turnover: 718902016.0, amplitude: 9.61, change_rate: -10.02, change_amount: -1.69, turnover_rate: 6.49 },
StockZhAHist { date: "1996-12-18T00:00:00.000", open: 15.28, close: 16.69, high: 16.69, low: 15.18, volume: 445380.0, turnover: 719400000.0, amplitude: 9.95, change_rate: 10.02, change_amount: 1.52, turnover_rate: 6.24 },
StockZhAHist { date: "1996-12-19T00:00:00.000", open: 17.01, close: 16.4, high: 17.9, low: 15.99, volume: 572946.0, turnover: 970124992.0, amplitude: 11.44, change_rate: -1.74, change_amount: -0.29, turnover_rate: 8.03 },
StockZhAHist { date: "1996-12-20T00:00:00.000", open: 16.19, close: 16.39, high: 16.68, low: 15.9, volume: 277758.0, turnover: 451471008.0, amplitude: 4.76, change_rate: -0.06, change_amount: -0.01, turnover_rate: 3.89 },
StockZhAHist { date: "1996-12-23T00:00:00.000", open: 16.58, close: 16.57, high: 16.89, low: 16.4, volume: 155369.0, turnover: 258260992.0, amplitude: 2.99, change_rate: 1.1, change_amount: 0.18, turnover_rate: 2.18 },
StockZhAHist { date: "1996-12-24T00:00:00.000", open: 16.58, close: 15.95, high: 16.73, low: 15.93, volume: 153168.0, turnover: 249324000.0, amplitude: 4.83, change_rate: -3.74, change_amount: -0.62, turnover_rate: 2.15 },
StockZhAHist { date: "1996-12-25T00:00:00.000", open: 16.1, close: 16.47, high: 16.7, low: 15.86, volume: 175391.0, turnover: 285567008.0, amplitude: 5.27, change_rate: 3.26, change_amount: 0.52, turnover_rate: 2.46 },
StockZhAHist { date: "1996-12-26T00:00:00.000", open: 16.68, close: 16.47, high: 16.78, low: 16.3, volume: 51516.0, turnover: 85214000.0, amplitude: 2.91, change_rate: 0.0, change_amount: 0.0, turnover_rate: 0.72 }...]
答案1
得分: 2
似乎是一个错误 - 如果您启用特性 dtype-struct，这个问题会消失，但您绝对不应该为了从 JSON 中读取而这样做。
作为一种解决方法，即使没有 dtype-struct，您也可以通过将数据转换为 JSON Lines 来使其工作。
use polars::prelude::{JsonReader, SerReader};
use serde::{Deserialize, Serialize};
use std::io::Cursor;
#[derive(Serialize, Deserialize, Debug)]
struct StockZhAHist {
date: String,
open: f64,
close: f64,
high: f64,
low: f64,
volume: f64,
turnover: f64,
amplitude: f64,
change_rate: f64,
change_amount: f64,
turnover_rate: f64,
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let data = vec![
StockZhAHist {
date: "1996-12-16T00:00:00.000".into(),
open: 16.86,
close: 16.86,
high: 16.86,
low: 16.86,
volume: 62442.0,
turnover: 105277000.0,
amplitude: 0.0,
change_rate: -10.22,
change_amount: -1.92,
turnover_rate: 0.87,
},
StockZhAHist {
date: "1996-12-17T00:00:00.000".into(),
open: 15.17,
close: 15.17,
high: 16.79,
low: 15.17,
volume: 463675.0,
turnover: 718902016.0,
amplitude: 9.61,
change_rate: -10.02,
change_amount: -1.69,
turnover_rate: 6.49,
},
StockZhAHist {
date: "1996-12-18T00:00:00.000".into(),
open: 15.28,
close: 16.69,
high: 16.69,
low: 15.18,
volume: 445380.0,
turnover: 719400000.0,
amplitude: 9.95,
change_rate: 10.02,
change_amount: 1.52,
turnover_rate: 6.24,
},
];
// JSON Lines 数据
let response = data
.into_iter()
.map(|d| serde_json::to_string(&d))
.collect::<Result<Vec<_>, _>>()?
.join("\n");
println!("{}", response);
// 转换为 Polars 数据框
let df = JsonReader::new(Cursor::new(response))
.infer_schema_len(Some(1000))
.with_json_format(polars::prelude::JsonFormat::JsonLines)
.finish()?;
println!("{:?}", df);
Ok(())
}
{"date":"1996-12-16T00:00:00.000","open":16.86,"close":16.86,"high":16.86,"low":16.86,"volume":62442.0,"turnover":105277000.0,"amplitude":0.0,"change_rate":-10.22,"change_amount":-1.92,"turnover_rate":0.87}
{"date":"1996-12-17T00:00:00.000","open":15.17,"close":15.17,"high":16.79,"low":15.17,"volume":463675.0,"turnover":718902016.0,"amplitude":9.61,"change_rate":-10.02,"change_amount":-1.69,"turnover_rate":6.49}
{"date":"1996-12-18T00:00:00.000","open":15.28,"close":16.69,"high":16.69,"low":15.18,"volume":445380.0,"turnover":719400000.0,"amplitude":9.95,"change_rate":10.02,"change_amount":1.52,"turnover_rate":6.24}
[/Users/ben/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-io-0.30.0/src/ndjson/core.rs:162] &data_type = Struct(
[
Field {
name: "date",
data_type: LargeUtf8,
is_nullable: true,
metadata: {},
},
Field {
name: "open",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "close",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "high",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "low",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "volume",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "turnover",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "amplitude",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "change_rate",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "change_amount",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "turnover_rate",
data_type: Float64,
is_nullable: true,
metadata: {},
},
],
)
shape: (3, 11)
┌─────────────────────────┬───────┬───────┬───────┬───┬───────────┬─────────────┬───────────────┬───────────────┐
│ date ┆ open ┆ close ┆ high ┆ … ┆ amplitude ┆ change_rate ┆ change_amount ┆ turnover_rate │
│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │
│ str ┆ f64 ┆ f64 ┆ f64 ┆ ┆ f64 ┆ f64 ┆ f64 ┆ f64 │
╞═════════════════════════╪═══════╪═══════╪═══════╪═══╪════════
<details>
<summary>英文:</summary>
Seems like a bug — this issue goes away if you enable feature `dtype-struct`, but you definitely shouldn't have to in order to read from json.
As a workaround, even without `dtype-struct`, you can get this to work by converting your data to JSON Lines.
```rust
use polars::prelude::{JsonReader, SerReader};
use serde::{Deserialize, Serialize};
use std::io::Cursor;
#[derive(Serialize, Deserialize, Debug)]
struct StockZhAHist {
date: String,
open: f64,
close: f64,
high: f64,
low: f64,
volume: f64,
turnover: f64,
amplitude: f64,
change_rate: f64,
change_amount: f64,
turnover_rate: f64,
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let data = vec![
StockZhAHist {
date: "1996-12-16T00:00:00.000".into(),
open: 16.86,
close: 16.86,
high: 16.86,
low: 16.86,
volume: 62442.0,
turnover: 105277000.0,
amplitude: 0.0,
change_rate: -10.22,
change_amount: -1.92,
turnover_rate: 0.87,
},
StockZhAHist {
date: "1996-12-17T00:00:00.000".into(),
open: 15.17,
close: 15.17,
high: 16.79,
low: 15.17,
volume: 463675.0,
turnover: 718902016.0,
amplitude: 9.61,
change_rate: -10.02,
change_amount: -1.69,
turnover_rate: 6.49,
},
StockZhAHist {
date: "1996-12-18T00:00:00.000".into(),
open: 15.28,
close: 16.69,
high: 16.69,
low: 15.18,
volume: 445380.0,
turnover: 719400000.0,
amplitude: 9.95,
change_rate: 10.02,
change_amount: 1.52,
turnover_rate: 6.24,
},
];
// JSON Lines data
let response = data
.into_iter()
.map(|d| serde_json::to_string(&d))
.collect::<Result<Vec<_>, _>>()?
.join("\n");
println!("{}", response);
// convert to polars dataframe
let df = JsonReader::new(Cursor::new(response))
.infer_schema_len(Some(1000))
.with_json_format(polars::prelude::JsonFormat::JsonLines)
.finish()?;
println!("{:?}", df);
Ok(())
}
{"date":"1996-12-16T00:00:00.000","open":16.86,"close":16.86,"high":16.86,"low":16.86,"volume":62442.0,"turnover":105277000.0,"amplitude":0.0,"change_rate":-10.22,"change_amount":-1.92,"turnover_rate":0.87}
{"date":"1996-12-17T00:00:00.000","open":15.17,"close":15.17,"high":16.79,"low":15.17,"volume":463675.0,"turnover":718902016.0,"amplitude":9.61,"change_rate":-10.02,"change_amount":-1.69,"turnover_rate":6.49}
{"date":"1996-12-18T00:00:00.000","open":15.28,"close":16.69,"high":16.69,"low":15.18,"volume":445380.0,"turnover":719400000.0,"amplitude":9.95,"change_rate":10.02,"change_amount":1.52,"turnover_rate":6.24}
[/Users/ben/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-io-0.30.0/src/ndjson/core.rs:162] &data_type = Struct(
[
Field {
name: "date",
data_type: LargeUtf8,
is_nullable: true,
metadata: {},
},
Field {
name: "open",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "close",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "high",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "low",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "volume",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "turnover",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "amplitude",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "change_rate",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "change_amount",
data_type: Float64,
is_nullable: true,
metadata: {},
},
Field {
name: "turnover_rate",
data_type: Float64,
is_nullable: true,
metadata: {},
},
],
)
shape: (3, 11)
┌─────────────────────────┬───────┬───────┬───────┬───┬───────────┬─────────────┬───────────────┬───────────────┐
│ date ┆ open ┆ close ┆ high ┆ … ┆ amplitude ┆ change_rate ┆ change_amount ┆ turnover_rate │
│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │
│ str ┆ f64 ┆ f64 ┆ f64 ┆ ┆ f64 ┆ f64 ┆ f64 ┆ f64 │
╞═════════════════════════╪═══════╪═══════╪═══════╪═══╪═══════════╪═════════════╪═══════════════╪═══════════════╡
│ 1996-12-16T00:00:00.000 ┆ 16.86 ┆ 16.86 ┆ 16.86 ┆ … ┆ 0.0 ┆ -10.22 ┆ -1.92 ┆ 0.87 │
│ 1996-12-17T00:00:00.000 ┆ 15.17 ┆ 15.17 ┆ 16.79 ┆ … ┆ 9.61 ┆ -10.02 ┆ -1.69 ┆ 6.49 │
│ 1996-12-18T00:00:00.000 ┆ 15.28 ┆ 16.69 ┆ 16.69 ┆ … ┆ 9.95 ┆ 10.02 ┆ 1.52 ┆ 6.24 │
└─────────────────────────┴───────┴───────┴───────┴───┴───────────┴─────────────┴───────────────┴───────────────┘
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库，让每个人都能够通过互相帮助和分享经验来进步。
评论