Skip to content

Commit a84ba74

Browse files
mariusarvintedxoigmnmzweilinCopilot
authored
Release 2025.11 (#327) (#24)
# 🎉 Major Updates - `ideas.init` module for robust C project discovery (including linked libraries) and idiomatic use of [`cargo workspace`](https://doc.rust-lang.org/book/ch14-03-cargo-workspaces.html) for the translated Rust project. C artifacts are now translated to self-contained Rust crates in the same workspace. - `ideas.translate_recurrent` module for symbol-by-symbol C-to-Rust translation using LLMs. The module is powerful enough to work across multi-artifact C projects and topologically sort symbols, assuming no name collisions or circular references exist. - `ideas.wrapper` module for LLM-based C FFI wrapper generation for Rust functions. The module uses `dspy.ChainOfThought` with custom feedback and `bindgen` signatures derived from the original C function to improve C ABI compatibility of the translated Rust functions. # 🎈 Minor Updates - Expanded Rust AST parsing functionality and added an `ensure_no_mangle` flag to `ideas.cover` to deterministically guarantee that translated functions get exported with the correct `#[unsafe(no_mangle)]` attribute. --------- Co-authored-by: Cory Cornelius <[email protected]> Co-authored-by: Weilin Xu <[email protected]> Co-authored-by: Copilot <[email protected]>
1 parent 6f1a00b commit a84ba74

32 files changed

+2181
-379
lines changed

‎CONTRIBUTING.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
### License
44

5-
<PROJECT NAME> is licensed under the terms in [LICENSE]<link to license file in repo>. By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms.
5+
IDEAS is licensed under the terms in [LICENSE](LICENSE). By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms.
66

77
### Sign your work
88

‎IDEAS.mk‎

Lines changed: 134 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
88
MAKEFILE_DIR := $(realpath $(dir $(MAKEFILE_PATH)))
9-
CARGO_TOML_CMAKE := ${MAKEFILE_DIR}/cargo_toml.cmake
9+
EXTRACT_INFO_CMAKE := ${MAKEFILE_DIR}/extract_info.cmake
1010
IDEAS_MAKEFILE := $(MAKEFILE_DIR)/IDEAS.mk
1111

1212
PROVIDER ?= hosted_vllm
@@ -18,6 +18,7 @@ TRANSLATION_DIR ?= translation.$(shell git --git-dir=${MAKEFILE_DIR}/.git rev-pa
1818
ifeq (${PROVIDER},hosted_vllm)
1919
override TRANSLATE_ARGS += model.base_url=${BASE_URL}
2020
override REPAIR_ARGS += model.base_url=${BASE_URL}
21+
override WRAPPER_ARGS += model.base_url=${BASE_URL}
2122
endif
2223
RUSTFLAGS ?= -Awarnings## Ignore Rust compiler warnings
2324
CFLAGS ?= -w## Ignore C compiler warnings
@@ -32,6 +33,13 @@ GREEN_COL := \033[1;32m
3233
PROJECT_C_FILES = $(shell jq -r 'map(.file) | .[] | @text' build-ninja/compile_commands.json)
3334
C_FILES = $(subst ${CURDIR}/test_case/,,${PROJECT_C_FILES})
3435
TEST_FILES := $(wildcard test_vectors/*.json)
36+
TARGETS := $(shell find build-ninja -maxdepth 1 -type f -executable -exec basename {} \; | cut -d. -f1 | sed -e "s/^lib//gi")
37+
ARTIFACTS := $(shell find build-ninja -maxdepth 1 -type f -executable -exec basename {} \;)
38+
ifeq (${TARGETS},)
39+
ifneq (${MAKECMDGOALS},cmake)
40+
$(error No TARGETS found! You need to run cmake!)
41+
endif
42+
endif
3543

3644
AFL_TAG = aflplusplus/aflplusplus:stable
3745
FUZZING_TIMEOUT ?= 60
@@ -40,9 +48,6 @@ FUZZING_TEST_VECTORS := $(subst :,\:, $(wildcard afl/out/default/queue/*))
4048

4149
CRATEIFY_BIN = ${MAKEFILE_DIR}/tools/crateify/target/debug/crateify
4250

43-
.PHONY: FORCE
44-
FORCE:
45-
4651

4752
# cmake
4853
cmake: build-ninja/build.log
@@ -51,41 +56,48 @@ build-ninja/translate.log: build-ninja/compile_commands.json
5156
@$(MAKE) --no-print-directory -f ${IDEAS_MAKEFILE} $(addprefix test_case/,$(addsuffix .i,${C_FILES}))
5257
@touch $@
5358

54-
.PRECIOUS: build-ninja/%
55-
build-ninja/%: build-ninja/CMakeCache.txt ;
56-
5759
.PRECIOUS: build-ninja/CMakeCache.txt
58-
build-ninja/CMakeCache.txt: test_case/CMakeLists.txt ${CARGO_TOML_CMAKE}
60+
build-ninja/CMakeCache.txt: test_case/CMakeLists.txt ${EXTRACT_INFO_CMAKE}
5961
@rm -rf build-ninja
6062
ifeq ($(wildcard CMakePresets.json),)
6163
cmake -S test_case -B build-ninja -G Ninja \
62-
-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${CARGO_TOML_CMAKE} \
63-
-DCMAKE_C_FLAGS="${CFLAGS}" \
64-
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
64+
-DCMAKE_BUILD_TYPE=Debug \
65+
-DCMAKE_C_FLAGS_DEBUG="-g -O0" \
66+
-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES="${EXTRACT_INFO_CMAKE}" \
67+
-DCMAKE_C_FLAGS="${CFLAGS}" \
68+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
6569
else
6670
cmake -S . --preset test \
67-
-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${CARGO_TOML_CMAKE} \
68-
-DCMAKE_C_FLAGS="${CFLAGS}" \
69-
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
71+
-DCMAKE_BUILD_TYPE=Debug \
72+
-DCMAKE_C_FLAGS_DEBUG="-g -O0" \
73+
-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES="${EXTRACT_INFO_CMAKE}" \
74+
-DCMAKE_C_FLAGS="${CFLAGS}" \
75+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
7076
endif
7177

7278
.PRECIOUS: build-ninja/compile_commands.json
7379
build-ninja/compile_commands.json: build-ninja/CMakeCache.txt ;
7480

7581
.PRECIOUS: build-ninja/build.log
76-
build-ninja/build.log: build-ninja/translate.log
82+
build-ninja/build.log: build-ninja/CMakeCache.txt
7783
ifeq ($(wildcard CMakePresets.json),)
7884
-cmake --build build-ninja --target all 2> $@
7985
else
8086
-cmake --build build-ninja --target all --preset test 2> $@
8187
endif
88+
@find build-ninja -maxdepth 1 -type f -executable | \
89+
xargs -I{} sh -c "nm --extern-only {} | \
90+
awk '{if (\$$2 == \"T\") print \$$NF}' | \
91+
grep -v ^_ > {}.symbols"
8292

8393
.PRECIOUS: test_case/%.c.i
8494
test_case/%.c.i: build-ninja/compile_commands.json
85-
$(shell cat build-ninja/compile_commands.json | \
95+
cat build-ninja/compile_commands.json | \
8696
jq -r '.[] | select(.file == "${CURDIR}/test_case/$*.c") | .command' | \
8797
sed -e 's/-o [^ ]*//g' | \
88-
xargs -I{} echo "{} -E -o $@")
98+
xargs -I{} echo "{} -E -o $@" | \
99+
sh
100+
89101

90102
# Add more tests from fuzzing. The procedure is
91103
# 1. Copy test input from the initial JSON test vectors;
@@ -168,95 +180,148 @@ test_vectors/%.json: afl/out/default/queue/%
168180

169181
# init
170182
.PHONY: init
171-
init: ${TRANSLATION_DIR}/Cargo.toml build-ninja/compile_commands.json ${CRATEIFY_BIN}
172-
@$(MAKE) --no-print-directory -f${IDEAS_MAKEFILE} $(addprefix ${TRANSLATION_DIR}/src/,$(patsubst src/%,%,$(patsubst %.c,%.rs,${C_FILES})))
173-
${CRATEIFY_BIN} ${TRANSLATION_DIR}/src
183+
init: $(patsubst %,${TRANSLATION_DIR}/%/init.log,${TARGETS}) ;
174184

175185
.PRECIOUS: ${TRANSLATION_DIR}/Cargo.toml
176-
${TRANSLATION_DIR}/Cargo.toml: build-ninja/Cargo.toml
177-
@mkdir -p $(@D)
178-
cp build-ninja/Cargo.toml $@
179-
180-
${TRANSLATION_DIR}/%.rs:
181-
@mkdir -p $(@D)
182-
echo 'fn main() {\n println!("Hello, world!");\n}' > $@
186+
${TRANSLATION_DIR}/Cargo.toml:
187+
mkdir -p $(@D)
188+
echo -n "[workspace]\nresolver = \"3\"" > $@
189+
190+
.PRECIOUS: ${TRANSLATION_DIR}/%/Cargo.toml
191+
${TRANSLATION_DIR}/%/Cargo.toml: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/lib%.so.type
192+
cargo new --quiet --lib --vcs=none $(@D)
193+
echo -n "\n[lib]\ncrate-type = [\"lib\", \"cdylib\"]" >> $@
194+
cargo add --quiet --manifest-path $@ --dev [email protected] [email protected] [email protected]
195+
cargo add --quiet --manifest-path $@ [email protected]
196+
197+
.PRECIOUS: ${TRANSLATION_DIR}/%/Cargo.toml
198+
${TRANSLATION_DIR}/%/Cargo.toml: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/%.type
199+
cargo new --quiet --bin --vcs=none $(@D)
200+
cargo add --quiet --manifest-path $@ --dev [email protected] [email protected] [email protected]
201+
cargo add --quiet --manifest-path $@ [email protected]
202+
203+
.PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.c
204+
${TRANSLATION_DIR}/%/src/lib.c: ${TRANSLATION_DIR}/%/init.log ;
205+
206+
.PRECIOUS: ${TRANSLATION_DIR}/%/init.log
207+
${TRANSLATION_DIR}/%/init.log: | ${TRANSLATION_DIR}/%/Cargo.toml build-ninja/lib%.so.type
208+
uv run python -m ideas.init filename=build-ninja/compile_commands.json \
209+
export_symbols=build-ninja/lib$*.so.symbols \
210+
source_priority=build-ninja/lib$*.so.sources \
211+
hydra.output_subdir=.init \
212+
hydra.run.dir=${TRANSLATION_DIR}/$*
213+
214+
.PRECIOUS: ${TRANSLATION_DIR}/%/src/main.c
215+
${TRANSLATION_DIR}/%/src/main.c: ${TRANSLATION_DIR}/%/init.log ;
216+
217+
.PRECIOUS: ${TRANSLATION_DIR}/%/init.log
218+
${TRANSLATION_DIR}/%/init.log: | ${TRANSLATION_DIR}/%/Cargo.toml build-ninja/%.type
219+
uv run python -m ideas.init filename=build-ninja/compile_commands.json \
220+
export_symbols=build-ninja/$*.symbols \
221+
source_priority=build-ninja/$*.sources \
222+
hydra.output_subdir=.init \
223+
hydra.run.dir=${TRANSLATION_DIR}/$*
183224

184225
.PRECIOUS: ${CRATEIFY_BIN}
185226
${CRATEIFY_BIN}:
186227
@cd ${MAKEFILE_DIR}/tools/crateify && cargo build
187228

188-
.PRECIOUS: runner/release/runner
189-
runner/release/runner: runner/Cargo.toml
190-
@cd runner && cargo build --release --target-dir .
191229

192230
# translate
193231
.PHONY: translate
194-
translate: ${TRANSLATION_DIR}/translate.log ;
195-
${TRANSLATION_DIR}/translate.log: build-ninja/compile_commands.json
196-
-uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} filename=build-ninja/compile_commands.json hydra.run.dir=${TRANSLATION_DIR} ${TRANSLATE_ARGS}
232+
translate: $(patsubst %,${TRANSLATION_DIR}/%/translate.log,${TARGETS}) ;
233+
234+
${TRANSLATION_DIR}/translate.log: $(patsubst %,${TRANSLATION_DIR}/%/translate.log,${TARGETS})
235+
cat $^ > $@
236+
237+
.PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.rs
238+
${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/translate.log ;
239+
240+
.PRECIOUS: ${TRANSLATION_DIR}/%/translate.log
241+
${TRANSLATION_DIR}/%/translate.log: | ${TRANSLATION_DIR}/%/src/lib.c build-ninja/compile_commands.json build-ninja/lib%.so.symbols build-ninja/lib%.so.sources
242+
-uv run python -m ideas.translate_recurrent model.name=${PROVIDER}/${MODEL} \
243+
filename=${TRANSLATION_DIR}/$*/src/lib.c \
244+
hydra.output_subdir=.translate \
245+
hydra.job.name=translate \
246+
hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS}
247+
248+
.PRECIOUS: ${TRANSLATION_DIR}/%/src/main.rs
249+
${TRANSLATION_DIR}/%/src/main.rs: ${TRANSLATION_DIR}/%/translate.log ;
250+
251+
.PRECIOUS: ${TRANSLATION_DIR}/%/translate.log
252+
${TRANSLATION_DIR}/%/translate.log: | ${TRANSLATION_DIR}/%/src/main.c build-ninja/compile_commands.json build-ninja/%.symbols build-ninja/%.sources
253+
-uv run python -m ideas.translate_recurrent model.name=${PROVIDER}/${MODEL} \
254+
filename=${TRANSLATION_DIR}/$*/src/main.c \
255+
hydra.output_subdir=.translate \
256+
hydra.job.name=translate \
257+
hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS}
258+
259+
260+
# wrapper
261+
.PHONY: wrapper
262+
wrapper: $(patsubst %,${TRANSLATION_DIR}/%/wrapper.log,${TARGETS}) ;
263+
264+
${TRANSLATION_DIR}/%/wrapper.log: ${TRANSLATION_DIR}/%/translate.log | build-ninja/lib%.so.symbols
265+
@mkdir -p $(@D)/src/wrapper
266+
-@cat build-ninja/lib$*.so.symbols | xargs -t -I{} bindgen --disable-header-comment --no-doc-comments --no-layout-tests $(@D)/src/lib.c --allowlist-function {} -o $(@D)/src/wrapper/{}.rs
267+
-@cat build-ninja/lib$*.so.symbols | xargs -t -I{} sed -zEe 's/\nunsafe extern "C" \{\s+(.*);\s+}/\n\#[unsafe(export_name = "{}")]\1 {\n unimplemented!();\n}/gi' -i $(@D)/src/wrapper/{}.rs
268+
-@cat build-ninja/lib$*.so.symbols | xargs -t -I{} sed -e 's/pub fn/pub extern "C" fn/gi' -i $(@D)/src/wrapper/{}.rs
269+
-@cat build-ninja/lib$*.so.symbols | xargs -t -I{} rustfmt ${@D}/src/wrapper/{}.rs
270+
-uv run python -m ideas.wrapper model.name=${PROVIDER}/${MODEL} \
271+
symbols=build-ninja/lib$*.so.symbols \
272+
cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \
273+
hydra.output_subdir=.wrapper \
274+
hydra.job.name=wrapper \
275+
hydra.run.dir=${TRANSLATION_DIR}/$* ${WRAPPER_ARGS}
276+
277+
${TRANSLATION_DIR}/%/wrapper.log: ${TRANSLATION_DIR}/%/translate.log | build-ninja/%.symbols ;
197278

198279

199280
# build
200281
.PHONY: build
201282
build: ${TRANSLATION_DIR}/build.log ;
202283

203-
.PRECIOUS: ${TRANSLATION_DIR}/build.log
204-
${TRANSLATION_DIR}/build.log: ${TRANSLATION_DIR}/translate.log ${TRANSLATION_DIR}/Cargo.toml ${CRATEIFY_BIN} FORCE
205-
${CRATEIFY_BIN} ${TRANSLATION_DIR}/src
284+
${TRANSLATION_DIR}/build.log: $(patsubst %,${TRANSLATION_DIR}/%/build.log,${TARGETS})
285+
cat $^ > $@
286+
287+
${TRANSLATION_DIR}/%/build.log: ${TRANSLATION_DIR}/%/wrapper.log
206288
-export RUSTFLAGS=${RUSTFLAGS} && cargo build --quiet --manifest-path $(@D)/Cargo.toml 2> $@
289+
@cat $@
290+
291+
${TRANSLATION_DIR}/target/debug/lib%.so: ${TRANSLATION_DIR}/%/build.log | build-ninja/lib%.so.type ;
292+
${TRANSLATION_DIR}/target/debug/%: ${TRANSLATION_DIR}/%/build.log | build-ninja/%.type ;
207293

208294

209-
# tests for executables
295+
# tests for TARGETS
210296
.PHONY: test
211297
test: ${TRANSLATION_DIR}/cargo_test.log ;
212298

213299
.PRECIOUS: ${TRANSLATION_DIR}/cargo_test.log
214-
${TRANSLATION_DIR}/cargo_test.log: ${TRANSLATION_DIR}/Cargo.toml \
215-
${TRANSLATION_DIR}/tests/test_cases.rs \
216-
${TRANSLATION_DIR}/build.log
217-
@if [ $$(stat -c %s ${TRANSLATION_DIR}/build.log) = 0 ]; then \
300+
${TRANSLATION_DIR}/cargo_test.log: ${TRANSLATION_DIR}/build.log $(patsubst %,${TRANSLATION_DIR}/%/tests/test_cases.rs,${TARGETS})
301+
if [ $$(stat -c %s ${TRANSLATION_DIR}/build.log) = 0 ]; then \
218302
cargo test --manifest-path ${TRANSLATION_DIR}/Cargo.toml --test test_cases | tee $@ ; \
219303
else \
220304
find test_vectors -name '*.json' -exec echo "test {} ... FAILED" \; | tee $@ ; \
221305
fi
222306

223-
.PRECIOUS: ${TRANSLATION_DIR}/tests/test_cases.rs
224-
${TRANSLATION_DIR}/tests/test_cases.rs: ${TEST_FILES}
307+
.PRECIOUS: ${TRANSLATION_DIR}/%/tests/test_cases.rs
308+
${TRANSLATION_DIR}/%/tests/test_cases.rs: ${TEST_FILES}
225309
@mkdir -p $(@D)
226-
-uv run python -m ideas.convert_tests $^ | rustfmt > $@
310+
-uv run python -m ideas.convert_tests ${TEST_FILES} | rustfmt > $@
227311

228312
.PRECIOUS: test_vectors/%.json
229313
test_vectors/%.json:
230314
$(error $@ not found)
231315

232-
# tests for C libraries
233-
.PHONY: test_libc
234-
test_libc: runner/test_libc.log ;
235-
236-
.PRECIOUS: runner/test_libc.log
237-
runner/test_libc.log: build-ninja/build.log \
238-
runner/release/runner
239-
find test_vectors -name '*.json' \
240-
| sort \
241-
| xargs -I {} sh -c './runner/release/runner lib -c ../{} -v' \
242-
| tee $@
243-
244-
# tests for Rust libraries
245-
.PHONY: test_librs
246-
test_librs: runner/test_librs.log ;
247-
248-
.PRECIOUS: runner/test_librs.log
249-
runner/test_librs.log: ${TRANSLATION_DIR}/build.log \
250-
runner/release/runner
251-
find test_vectors -name '*.json' \
252-
| sort \
253-
| xargs -I {} sh -c './runner/release/runner -b ${TRANSLATION_DIR}/target/debug lib -c ../{} -v' \
254-
| tee $@
255316

256317
# repair
257318
.PHONY: repair
258-
repair: ${TRANSLATION_DIR}/translate.log ${TRANSLATION_DIR}/Cargo.toml ${TRANSLATION_DIR}/tests/test_cases.rs
259-
-uv run python -m ideas.repair model.name=${PROVIDER}/${MODEL} cargo_toml=${TRANSLATION_DIR}/Cargo.toml ${REPAIR_ARGS}
319+
repair: ${TRANSLATION_DIR}/translate.log \
320+
${TRANSLATION_DIR}/Cargo.toml \
321+
${TRANSLATION_DIR}/tests/test_cases.rs
322+
-uv run python -m ideas.repair model.name=${PROVIDER}/${MODEL} \
323+
cargo_toml=${TRANSLATION_DIR}/Cargo.toml \
324+
${REPAIR_ARGS}
260325

261326
# clean
262327
.PHONY: clean

0 commit comments

Comments
 (0)