Java、Python、C混合编程

编译后的文件

window：

.dll 动态
.lib 静态

Linux

.o 对应 Windows上的 .obj
.a 多个 .o 的合并
.so 对应 Windows上的 .dll

MacOS

动态 .dylib
静态 .a

【ctypes】python 调 C 编译后的动态库

优缺点：

优点：无需处理 Python 版本
缺点：不能多线程

步骤

写 c 文件，保存为 file.c

#include<stdio.h>
void add()
{
int sum=0;
for(int i=0;i<1000;i++)
sum+=i;
printf("sum:%d\n",sum);
}

把 file.c 文件编译为 file.so：gcc file.c -shared -o file.so

在Python中导入和使用 .so

from ctypes import cdll
clib = cdll.LoadLibrary("./file.so")
clib.add()

如何同时支持 windows/Linux : https://www.bilibili.com/video/BV17y4y1W7BY?p=3

数据类型

ctypes 类型	C 类型	Python 数据类型
c_bool	_Bool	bool (1)
c_char	char	单字符字节串对象
c_wchar	wchar_t	单字符字符串
c_byte	char	int
c_ubyte	unsigned char	int
c_short	short	int
c_ushort	unsigned short	int
c_int	int	int
c_uint	unsigned int	int
c_long	long	int
c_ulong	unsigned long	int
c_longlong	__int64 或 long long	int
c_ulonglong	unsigned __int64 或 unsigned long long	int
c_size_t	size_t	int
c_ssize_t	ssize_t 或 Py_ssize_t	int
c_float	float	float
c_double	double	float
c_longdouble	long double	float
c_char_p	char * (NUL terminated)	字节串对象或 None
c_wchar_p	wchar_t * (NUL terminated)	字符串或 None
c_void_p	void *	int 或 None

例子：

// c 
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

int my_c_fun(int x1, float x2, char *x3 // 指向字符串的指针
        , int *x4 // 指向 array 的指针
        , char **x5 //指向 array of string 的指针
        , int *x6 // 指向一个 int 类型的指针，在函数中修改它指向的值
        , char *x7 // 指向一个 string 类型的指针，在函数中修改它指向的值
) {
    printf("Get input :\n");
    printf("x1 = %d\n", x1);
    printf("x2 = %f\n", x2);
    printf("x3 = %s\n", x3);

    printf("x4 = ");
    for (int i = 0; i < 5; i++) {
        printf(" %d", x4[i]);
    }
    printf("\n");

    printf("x5 = ");
    for (int i = 0; i < 3; i++) {
        printf(" %s", x5[i]);
    }
    printf("\n");

    // 修改指针指向的值
    *x6 = 8;
    printf("%s\n", x7);
    strcpy(x7, "你好");

    return x1 + (int) x2 + (int) strlen(x3) + x4[0];
}


// python

from ctypes import cdll

clib = cdll.LoadLibrary("./file.so")
import ctypes

clib.my_c_fun.argtypes = [
    ctypes.c_int,
    ctypes.c_float,
    ctypes.c_char_p,
    ctypes.POINTER(ctypes.c_int),
    ctypes.POINTER(ctypes.c_char_p),
    ctypes.POINTER(ctypes.c_int),
    ctypes.c_char_p,
]

clib.my_c_fun.restype = ctypes.c_int

int_lst = [1, 2, 3, 4, 5]
x4 = (ctypes.c_int * len(int_lst))()
x4[:] = int_lst

# Pass a list of string
str_lst = ["hello", "world", "!", "\0"]
x5 = (ctypes.c_char_p * len(str_lst))()
x5[:] = [i.encode("utf-8") for i in str_lst]

x6 = ctypes.c_int()
x7 = ctypes.create_string_buffer(100)
x7.value = "nihao".encode('utf-8')  # 也先可以赋予一个值

res = clib.my_c_fun(
    1  # 默认 int 类型，无需转化
    , 1.2  # 传入 float
    , "hello".encode('utf-8')  # 传入字符串 char *
    , x4
    , x5
    , ctypes.byref(x6)
    , x7
)

print('res:', res)
print("x6:", x6.value)
print("x7:", x7.value.decode('utf-8'))

int my_c_fun(int x1, float x2, char *x3, int *x4, char **x5) {
    printf("Get input :\n");
    printf("x1 = %d\n", x1);
    printf("x2 = %f\n", x2);
    printf("x3 = %s\n", x3);

    printf("x4 = ");
    for (int i = 0; i < 5; i++) {
        printf(" %d", x4[i]);
    }
    printf("\n");

    printf("x5 = ");
    for (int i = 0; i < 3; i++) {
        printf(" %s", x5[i]);
    }
    printf("\n");


    return x1 + (int) x2 + (int) strlen(x3) + x4[0];
}

char *my_c_fun2() {
    char *p = malloc(20);
    strcpy(p, "hello");
    return p;
}


int *my_c_fun3() {
    int *p = malloc(20 * sizeof(int));
    for (int i = 0; i < 20; i++) {
        p[i] = 30 - i;
    }
    return p;
}

// my_c_fun3 对应的内存应当及时 free。如果这里 free 了，python脚本里面的返回值也就free了
void my_c_fun3_free(int *p) {
    free(p);
}


char **my_c_fun4() {
    char **p = malloc(5 * sizeof(char *));
    p[0] = "hello";
    p[1] = "world";
    return p;
}

用 python 去调用上面那个 C

import ctypes

clib = ctypes.cdll.LoadLibrary("/Users/guofei/c_algorithm/main.o")

# Pass a list of int
int_lst = [1, 2, 3, 4, 5]
int_arr = (ctypes.c_int * len(int_lst))()
int_arr[:] = int_lst

# Pass a list of string
str_lst = ["hello", "world", "!", "\0"]
str_arr = (ctypes.c_char_p * len(str_lst))()
str_arr[:] = [i.encode("utf-8") for i in str_lst]

x = clib.my_c_fun(1  # 默认 int 类型，无需转化
                  , ctypes.c_float(1.2)  # 传入 float
                  , ctypes.c_char_p("hello".encode('utf-8'))  # 传入字符串 char *
                  , int_arr # 传入 int *
                  , str_arr # 传入 char **
                  )
print('res:', x)

# 返回一个字符串
clib.my_c_fun2.restype = ctypes.c_char_p
x = clib.my_c_fun2()
print('res:', x)

# 返回 list of int
clib.my_c_fun3.restype = ctypes.POINTER(ctypes.c_int * 20)
x = clib.my_c_fun3()
print('res:', list(x.contents))

# 注意：用完记得 free，free之后，x 的内容也变了（内存被释放了）
clib.my_c_fun3_free(x)

# 返回 list of string
clib.my_c_fun4.restype = ctypes.POINTER(ctypes.c_char_p * 2)
x = clib.my_c_fun4()
print('res:', list(x.contents))

# 这个不知道什么用
# clib.my_c_fun2.argtypes = (ctypes.c_char_p,)

Python 调 Rust

Rust混合编程

https://www.guofei.site/2022/08/28/rust2.html#混合编程

【JPype1】Python 调 Java

安装

pip install JPype1

示例java脚本

package com.guofei9987.learn_java;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class JavaClass {

    public static String tstStatic(String a) {
        return "调用了静态方法，入参：" + a;
    }

    public List<String> my_java_fun(int x1, String x2, List<Integer> x3, List<String> x4) {
        System.out.println("x1 = " + x1);
        System.out.println("x2 = " + x2);
        System.out.println("x3 = " + x3);
        System.out.println("x4 = " + x4);

        ArrayList<String> res = new ArrayList<String>(Arrays.asList("" + x1, x2)) {
        };
        res.addAll(x4);
        return res;
    }

    public String my_java_fun(int a) {
        return "调用了重载的方法，入参：" + a;
    }
}

Python 调用 Java

import jpype
import os

jarpath = os.path.join(os.path.abspath("."), "/Users/guofei9987/learn_java/target/learn_java-1.0-SNAPSHOT.jar")
jvmPath = jpype.getDefaultJVMPath()
jpype.startJVM(jvmPath, "-ea", "-Djava.class.path=%s" % (jarpath))

# step1：加载类
javaClass = jpype.JClass("com.guofei9987.learn_java.JavaClass")

# step2：实例化java对象
javaInstance = javaClass()

# step3：调用java方法
javaClass.tstStatic("Hello")
# >>>'调用了静态方法，入参：Hello'


output = javaInstance.my_java_fun(
    1  # 传入 int
    , "hello"  # 传入 String
    , jpype.java.util.ArrayList([1, 2, 3])  # 传入 ArrayList<int>
    , jpype.java.util.ArrayList(["hello", "world"]))  # 传入 ArrayList<String>
print(list(output))

output2 = javaInstance.my_java_fun(1)
print(output2)

# step4：关闭jvm
jpype.shutdownJVM()

关于变量类型

java.lang.String
- 把 java.lang.String 转为 Python 的 str 类型：str(java_string)
- 把 Python 的 str 转为 java.lang.String：jpype.JClass("java.lang.String")("Hello, World!")

【PMML】Java调 sklearn 算法模型

描述

Python 训练算法模型，保存为 PMML 文件。然后 Java 调用 PMML 文件做预测，这样线上就变成纯 JAVA 项目了。

使用

python 训练模型

from sklearn import tree
from sklearn.datasets import load_iris
from sklearn2pmml.pipeline import PMMLPipeline
from sklearn2pmml import sklearn2pmml

iris = load_iris()
X, y = iris.data, iris.target
pipeline = PMMLPipeline([("classifier", tree.DecisionTreeClassifier())])  # 用决策树分类
pipeline.fit(X, y)
sklearn2pmml(pipeline, "iris.pmml", with_repr=True)

java 调用模型做预测

package com.alipay.tianqian;

import org.dmg.pmml.FieldName;
import org.jpmml.evaluator.*;
import org.xml.sax.SAXException;

import javax.xml.bind.JAXBException;
import java.io.*;
import java.util.*;


public class TestPmml {

    public static void main(String[] args) throws JAXBException, SAXException, IOException {
        TestPmml obj = new TestPmml();

        Evaluator model = new LoadingModelEvaluatorBuilder()
                .load(new File("src/iris.pmml"))
                .build();

        List<Map<String, Object>> inputs = new ArrayList<>();
        inputs.add(obj.getRawMap(5.1, 3.5, 1.4, 0.2));
        inputs.add(obj.getRawMap(4.9, 3, 1.5, 4));
        for (Map<String, Object> input : inputs) {
            Map<String, Object> output = obj.predict(model, input);
            System.out.println("X=" + input + " -> y=" + output.get("y"));
        }
    }



    private Map<String, Object> getRawMap(Object a, Object b, Object c, Object d) {
        Map<String, Object> data = new HashMap<String, Object>();
        data.put("x1", a);
        data.put("x2", b);
        data.put("x3", c);
        data.put("x4", d);
        return data;
    }

    /**
     * 运行模型得到结果。
     */
    private Map<String, Object> predict(Evaluator evaluator, Map<String, Object> data) {
        Map<FieldName, FieldValue> input = getFieldMap(evaluator, data);
        Map<String, Object> output = evaluate(evaluator, input);
        return output;
    }

    /**
     * 把原始输入转换成PMML格式的输入。
     */
    private Map<FieldName, FieldValue> getFieldMap(Evaluator evaluator, Map<String, Object> input) {
        List<InputField> inputFields = evaluator.getInputFields();
        Map<FieldName, FieldValue> map = new LinkedHashMap<FieldName, FieldValue>();
        for (InputField field : inputFields) {
            FieldName fieldName = field.getName();
            Object rawValue = input.get(fieldName.getValue());
            FieldValue value = field.prepare(rawValue);
            map.put(fieldName, value);
        }
        return map;
    }

    /**
     * 运行模型得到结果。
     */
    private Map<String, Object> evaluate(Evaluator evaluator, Map<FieldName, FieldValue> input) {
        Map<FieldName, ?> results = evaluator.evaluate(input);
        List<TargetField> targetFields = evaluator.getTargetFields();
        Map<String, Object> output = new LinkedHashMap<String, Object>();
        for (TargetField field : targetFields) {
            FieldName fieldName = field.getName();
            Object value = results.get(fieldName);
            if (value instanceof Computable) {
                Computable computable = (Computable) value;
                value = computable.getResult();
            }
            output.put(fieldName.getValue(), value);
        }
        return output;
    }
}

0x00_读论文 8

0x11_算法平台 15

0x12_Pandas与numpy 12

0x13_特征工程 4

0x21_有监督学习 21

0x22_上世纪神经网络 10

0x23_深度学习 7

0x24_NLP 13

0x25_CV 11

0x26_torch 11

0x31_降维 10

0x32_聚类 5

0x33_图模型 9

0x41_统计模型 9

0x42_概率论 7

0x43_时间序列 10

0x44_随机过程 2

0x51_代数与分析 14

0x52_方程 2

0x53_复分析与积分变换 8

0x55_数值计算 7

0x56_最优化 11

0x58_密码学 13

0x59_应用数学 9

0x60_启发式算法 8

0x70_可视化 13

0x80_数据结构与算法 17

0xa0_蒙特卡洛方法 6

0xb0_Python语法 18

0xd0_设计模式 6