Repository navigation
Expand file tree
/
Copy pathPass_FunctionCalling.cpp
More file actions
1424 lines (1254 loc) · 59.8 KB
/
Copy pathPass_FunctionCalling.cpp
File metadata and controls
1424 lines (1254 loc) · 59.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include "Pass_FunctionCalling.h"
#include "Utils.h"
#include <clang/AST/ASTConsumer.h>
#include <clang/AST/ASTContext.h>
#include <clang/AST/Attrs.inc>
#include <clang/AST/Expr.h>
#include <clang/AST/RecursiveASTVisitor.h>
#include <clang/AST/Stmt.h>
#include <clang/AST/TypeBase.h>
#include <clang/Basic/LLVM.h>
#include <clang/Basic/SourceLocation.h>
#include <clang/Basic/Specifiers.h>
#include <clang/Frontend/CompilerInstance.h>
#include <clang/Lex/Lexer.h>
#include <clang/Rewrite/Core/Rewriter.h>
#include <clang/Tooling/Core/Replacement.h>
#include <llvm/ADT/DenseMap.h>
#include <llvm/ADT/MapVector.h>
#include <llvm/ADT/ScopeExit.h>
#include <llvm/ADT/SmallVector.h>
#include <llvm/ADT/StringExtras.h>
#include <llvm/ADT/StringRef.h>
#include <llvm/Support/Casting.h>
#include <llvm/Support/Error.h>
#include <llvm/Support/ErrorHandling.h>
#include <llvm/Support/FormatVariadic.h>
#include <llvm/Support/raw_ostream.h>
#include <algorithm>
#include <cstddef>
#include <optional>
#include <ranges>
#include <string>
#include <utility>
using namespace clang;
using namespace clang::tooling;
extern llvm::cl::opt<bool> opt_treat_external_functions_as_non_ffi;
extern llvm::cl::list<std::string> function_rewrite_extra_headers;
namespace pancake::pass_inject_memcpy_polyfill {
namespace {
const char *memcpy_polyfill_code = R"(
#include <stdint.h>
static inline uint{0}_t __c2pnk_memcpy(uint8_t *dest, uint8_t *src, uint{0}_t len) {{
while (len > 0UL) {{
*dest = *src;
dest = dest + 1UL;
src = src + 1UL;
len = len - 1UL;
}
return 0UL;
}
)";
}
auto Consumer::HandleTranslationUnit(ASTContext &Ctx) -> void {
const auto &sm = Ctx.getSourceManager();
if (auto error = ps_ctx.replacements.add({sm, sm.getLocForStartOfFile(sm.getMainFileID()), 0,
llvm::formatv(memcpy_polyfill_code, GetPointerWidth(Ctx)).str()})) {
ps_ctx.error = llvm::joinErrors(CreateRuntimeError("Add replacement conflict"), std::move(error));
ps_ctx.SetControl(StageControl::NextFile);
return;
}
ps_ctx.SetControl(StageControl::Continue);
}
} // namespace pancake::pass_inject_memcpy_polyfill
namespace pancake::pass_function_calling {
namespace {
// all rewrites for a non-ffi function
// contains definition rewrites, return rewrites, declaration rewrites, and call expr rewrites
// also captures the more invasive rewrites done for external entry points
// needs to preserve insertion order because we apply the innermost rewrites first
using NonFFIFunctionRewrites = llvm::MapVector<FunctionDecl *, Replacements>;
// ffi function rewrites
// contains the ffi wrappers, and call expression rewrites
// needs to preserve insertion order because we apply the innermost rewrites first
using FFIFunctionRewrites = llvm::MapVector<FunctionDecl *, Replacements>;
// ffi variadic function rewrites
// these are keyed be the call expr * to provide something unique every pass
using FFIVariadicFunctionRewrites = llvm::MapVector<CallExpr *, Replacements>;
namespace CollectFunctionInfo {
struct WorkerData {
ASTContext &Ctx;
StageContext &ps_ctx;
struct ReturnInfo {
QualType return_type; // COULD be void
std::optional<llvm::SmallString<32>> hoisted_name;
explicit ReturnInfo(QualType return_type, std::optional<llvm::SmallString<32>> hoisted_name = std::nullopt)
: return_type(return_type), hoisted_name(std::move(hoisted_name)) {}
};
struct ParamInfo {
ParmVarDecl *param_decl; // type COULD be void
std::optional<llvm::SmallString<32>> hoisted_name;
explicit ParamInfo(ParmVarDecl *param_decl, std::optional<llvm::SmallString<32>> hoisted_name = std::nullopt)
: param_decl(param_decl), hoisted_name(std::move(hoisted_name)) {}
};
struct FunctionInfo {
std::optional<ReturnInfo> return_info;
llvm::SmallVector<ParamInfo, 16> param_infos;
bool needs_rewriting = false;
bool turn_return_from_void_to_uint = false;
explicit FunctionInfo(std::optional<ReturnInfo> return_info, llvm::SmallVector<ParamInfo, 16> param_infos,
bool needs_rewriting, bool turn_return_from_void_to_uint)
: return_info(std::move(return_info)), param_infos(std::move(param_infos)), needs_rewriting(needs_rewriting),
turn_return_from_void_to_uint(turn_return_from_void_to_uint) {}
};
using FunctionMap = llvm::DenseMap<FunctionDecl *, FunctionInfo>;
FunctionMap &function_map;
llvm::Error error = llvm::Error::success();
size_t tmp_var_counter = 0;
};
class Worker : public RecursiveASTVisitor<Worker> {
struct WorkerData &data;
public:
explicit Worker(struct WorkerData &data) : data(data) {}
// process all outer record decls first
static auto shouldTraversePostOrder() -> bool { return false; }
auto GetTempVarName(std::string hint) -> auto {
return llvm::formatv("__c2pnk_{0}_{1}_{2}_{3}", hint, data.ps_ctx.stage_index, data.ps_ctx.attempt_index,
data.tmp_var_counter++);
}
auto TraverseFunctionDecl(FunctionDecl *func_decl) -> bool {
if (data.error) {
return false;
}
auto &sm = data.Ctx.getSourceManager();
if (!sm.isInMainFile(sm.getSpellingLoc(func_decl->getBeginLoc())) || !func_decl->isThisDeclarationADefinition()) {
return true;
}
if (func_decl->isVariadic()) {
data.error = CreateRuntimeError(
llvm::formatv("\n at {0}\nNon-FFI FunctionDecl {1} is variadic, which is not supported by pancake",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName()));
return false;
}
auto policy = data.Ctx.getPrintingPolicy();
auto it = data.function_map.find(func_decl);
if (it == data.function_map.end()) {
// process the function decl
auto return_type = func_decl->getReturnType();
if ((return_type->isIntegerType() || return_type->isPointerType()) &&
data.Ctx.getTypeSize(return_type) > GetPointerWidth(data.Ctx)) {
data.error = CreateRuntimeError(
llvm::formatv("\n at {0}\nFunctionDecl {1} has integer return type {2} larger than pointer width",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName(),
return_type.getAsString(policy)));
return false;
}
bool needs_rewriting = false;
std::optional<WorkerData::ReturnInfo> return_info = std::nullopt;
if (!return_type->isIntegerType() && !return_type->isPointerType() && !return_type->isVoidType()) {
// hoist the return type
auto hoisted_name = llvm::SmallString<32>(llvm::formatv("__c2pnk_{0}_return", func_decl->getName()));
return_info = WorkerData::ReturnInfo(return_type);
return_info.value().hoisted_name = std::move(hoisted_name);
needs_rewriting = true;
}
bool turn_return_from_void_to_uint = false;
if (return_type->isVoidType()) {
// function needs to return 0UL as per pancake rules
turn_return_from_void_to_uint = true;
}
llvm::SmallVector<WorkerData::ParamInfo, 16> param_infos;
for (const auto [i, param_decl] : func_decl->parameters() | std::views::enumerate) {
auto param_type = param_decl->getType();
if ((param_type->isIntegerType() || param_type->isPointerType()) &&
data.Ctx.getTypeSize(param_type) > GetPointerWidth(data.Ctx)) {
data.error = CreateRuntimeError(std::move(llvm::formatv(
"\n at {0}\nFunctionDecl {1} has integer parameter {2} of type {3} larger than pointer width",
param_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName(),
param_decl->getName(), param_type.getAsString(policy))));
return false;
}
auto param_info = WorkerData::ParamInfo(param_decl);
std::optional<llvm::SmallString<32>> hoisted_name = std::nullopt;
if (!param_type->isIntegerType() && !param_type->isPointerType() && !param_type->isVoidType()) {
// hoist the parameter type
hoisted_name = llvm::SmallString<32>(
llvm::formatv("__c2pnk_{0}_param{1}_{2}", func_decl->getName(), i, param_decl->getName()));
needs_rewriting = true;
}
param_info.hoisted_name = std::move(hoisted_name);
param_infos.push_back(std::move(param_info));
}
data.function_map.insert({func_decl, WorkerData::FunctionInfo(std::move(return_info), std::move(param_infos),
needs_rewriting, turn_return_from_void_to_uint)});
}
return true;
}
};
} // namespace CollectFunctionInfo
/*
Annotation types:
[[clang::annotate("__c2pnk_rewritten_non_ffi_function")]]: marks a function signature as being rewritten due to passing
structs by value.
[[clang::annotate("__c2pnk_external_entry_point")]]: marks a function as being an external entry point for the pancake
runtime. Function definitions with this signature will not be touched. However, their body will be replaced with a call
to a new function which does the same thing but adheres to pancake calling convention. During the final translation
stage, the function marked with this annotation will be removed entirely.
*/
namespace RewriteFunctions {
using FunctionInfo = CollectFunctionInfo::WorkerData::FunctionInfo;
using FunctionMap = CollectFunctionInfo::WorkerData::FunctionMap;
struct FFIFunctionWrapperParamInfo {
ParmVarDecl *original_param_decl;
size_t original_param_index;
std::optional<std::pair<size_t, size_t>> offset_and_size;
// if not present, then it's on the fast path
// otherwise, the first element is the offset in bytes from the start of the input buffer, and the second element is
// the size in bytes
explicit FFIFunctionWrapperParamInfo(ParmVarDecl *original_param_decl, size_t original_param_index,
std::optional<std::pair<size_t, size_t>> offset_and_size = std::nullopt)
: original_param_decl(original_param_decl), original_param_index(original_param_index),
offset_and_size(std::move(offset_and_size)) {}
};
struct FFIFunctionWrapperReturnInfo {
QualType original_return_type;
std::optional<size_t> size_in_bytes;
explicit FFIFunctionWrapperReturnInfo(QualType original_return_type,
std::optional<size_t> size_in_bytes = std::nullopt)
: original_return_type(original_return_type), size_in_bytes(size_in_bytes) {}
// "fast path" (meaning no memcpy but global var still needed)
auto IsFastPath() const -> bool { return !size_in_bytes.has_value(); }
};
struct FFIFunctionWrapperInfo {
std::array<std::optional<FFIFunctionWrapperParamInfo>, 4> param_infos;
llvm::SmallVector<FFIFunctionWrapperParamInfo> non_fast_path_param_infos;
llvm::SmallVector<std::pair<std::optional<size_t>, std::optional<size_t>>> param_map;
size_t non_fastpath_input_buf_size; // compile-time constant, hardcoded into generated code, no slot
FFIFunctionWrapperReturnInfo return_info;
explicit FFIFunctionWrapperInfo(std::array<std::optional<FFIFunctionWrapperParamInfo>, 4> param_infos,
llvm::SmallVector<FFIFunctionWrapperParamInfo> non_fast_path_param_infos,
llvm::SmallVector<std::pair<std::optional<size_t>, std::optional<size_t>>> param_map,
size_t non_fastpath_input_buf_size, FFIFunctionWrapperReturnInfo return_info)
: param_infos(std::move(param_infos)), non_fast_path_param_infos(std::move(non_fast_path_param_infos)),
param_map(std::move(param_map)), non_fastpath_input_buf_size(non_fastpath_input_buf_size),
return_info(return_info) {}
// Only two possible reserved slots now: output_buf, then input_buf. Lengths are compile-time
// constants (return_info.size_in_bytes / non_fastpath_input_buf_size) baked directly into the
// generated wrapper body, so they never occupy a slot.
auto HasOutputBufSlot() const -> bool { return !return_info.original_return_type->isVoidType(); }
auto HasInputSlot() const -> bool { return !non_fast_path_param_infos.empty(); }
auto OutputBufSlotIndex() const -> std::optional<size_t> {
if (!HasOutputBufSlot())
return std::nullopt;
return 0;
}
auto InputBufSlotIndex() const -> std::optional<size_t> {
if (!HasInputSlot())
return std::nullopt;
return HasOutputBufSlot() ? 1 : 0;
}
};
struct WorkerData {
ASTContext &Ctx;
StageContext &ps_ctx;
FunctionMap &function_map;
NonFFIFunctionRewrites &non_ffi_rewrites;
FFIFunctionRewrites &ffi_rewrites;
FFIVariadicFunctionRewrites &ffi_variadic_rewrites;
llvm::DenseMap<const FunctionDecl *, FFIFunctionWrapperInfo> ffi_function_info;
llvm::Error error = llvm::Error::success();
size_t tmp_var_counter = 0;
FunctionDecl *current_function_decl = nullptr;
explicit WorkerData(ASTContext &Ctx, StageContext &ps_ctx, FunctionMap &function_map,
NonFFIFunctionRewrites &non_ffi_rewrites, FFIFunctionRewrites &ffi_rewrites,
FFIVariadicFunctionRewrites &ffi_variadic_rewrites)
: Ctx(Ctx), ps_ctx(ps_ctx), function_map(function_map), non_ffi_rewrites(non_ffi_rewrites),
ffi_rewrites(ffi_rewrites), ffi_variadic_rewrites(ffi_variadic_rewrites) {}
};
class Worker : public RecursiveASTVisitor<Worker> {
struct WorkerData &data;
public:
explicit Worker(struct WorkerData &data) : data(data) {}
// process all outer record decls first
static auto shouldTraversePostOrder() -> bool { return false; }
auto GetTempVarName(std::string hint) -> auto {
return llvm::formatv("__c2pnk_{0}_{1}_{2}_{3}", hint, data.ps_ctx.stage_index, data.ps_ctx.attempt_index,
data.tmp_var_counter++);
}
auto PrintType(const QualType ty, const llvm::StringRef var_name) const -> std::string {
std::string s;
llvm::raw_string_ostream os(s);
ty.print(os, data.Ctx.getPrintingPolicy(), var_name);
os.flush();
return s;
}
static auto GetStorageClassSpecifierString(StorageClass sc) -> auto {
switch (sc) {
case SC_None:
return "";
case SC_Extern:
return "extern";
case SC_Static:
return "static";
case SC_PrivateExtern:
return "__private_extern__";
case SC_Auto:
return "auto";
case SC_Register:
return "register";
}
}
auto ConstructFunctionWrapperInfo(const FunctionDecl *func_decl) const -> FFIFunctionWrapperInfo {
std::array<std::optional<FFIFunctionWrapperParamInfo>, 4> param_infos{};
llvm::SmallVector<FFIFunctionWrapperParamInfo> non_fast_path_param_infos;
llvm::SmallVector<std::pair<std::optional<size_t>, std::optional<size_t>>> param_map;
std::optional<FFIFunctionWrapperReturnInfo> return_info;
auto pointer_width = GetPointerWidth(data.Ctx);
auto fits_fastpath = [&](QualType type) -> bool {
auto size = data.Ctx.getTypeSize(type) / 8;
return (type->isIntegerType() || type->isPointerType()) && size <= pointer_width;
};
// process return type first
auto return_type = func_decl->getReturnType();
if (return_type->isVoidType()) {
return_info = FFIFunctionWrapperReturnInfo(return_type);
} else {
auto return_type_size = data.Ctx.getTypeSize(return_type) / 8;
if (fits_fastpath(return_type)) {
// fits directly: no memcpy needed, just a typed store through the output pointer
return_info = FFIFunctionWrapperReturnInfo(return_type);
} else {
// too big: needs a memcpy into the caller-provided output buffer
return_info = FFIFunctionWrapperReturnInfo(return_type, return_type_size);
}
}
// reserved slots always sit at a fixed prefix: output_buf [, input_buf].
// lengths are compile-time constants, so they never take a slot.
size_t fastpath_index_counter = 0;
if (!return_type->isVoidType()) {
fastpath_index_counter++; // output_buf
}
auto num_params = func_decl->getNumParams();
size_t remaining_slots_after_output = 4 - fastpath_index_counter;
bool needs_input_buffer = num_params > remaining_slots_after_output;
if (!needs_input_buffer) {
for (const auto ¶m_decl : func_decl->parameters()) {
if (!fits_fastpath(param_decl->getType())) {
needs_input_buffer = true;
break;
}
}
}
if (needs_input_buffer) {
fastpath_index_counter++; // input_buf
}
// place each parameter: fastpath-eligible params fill whatever slots remain, in order
// anything left over (type-ineligible, or the fastpath slots ran out) goes into the
// non-fastpath input buffer, with its offset/size recorded and padded to the param's
// required alignment so structs (and anything else with alignment > 1) land correctly.
size_t current_input_buf_offset = 0;
for (const auto [i, param_decl] : func_decl->parameters() | std::views::enumerate) {
auto param_type = param_decl->getType();
auto param_type_size = data.Ctx.getTypeSize(param_type) / 8;
if (fits_fastpath(param_type) && fastpath_index_counter < 4) {
param_infos.at(fastpath_index_counter) = FFIFunctionWrapperParamInfo(param_decl, static_cast<size_t>(i));
param_map.push_back(std::make_pair(fastpath_index_counter, std::nullopt));
fastpath_index_counter++;
} else {
auto param_align = data.Ctx.getTypeAlign(param_type) / 8; // bits -> bytes
current_input_buf_offset = llvm::alignTo(current_input_buf_offset, param_align);
non_fast_path_param_infos.push_back(FFIFunctionWrapperParamInfo(
param_decl, static_cast<size_t>(i), std::make_pair(current_input_buf_offset, param_type_size)));
param_map.push_back(std::make_pair(std::nullopt, non_fast_path_param_infos.size() - 1));
current_input_buf_offset += param_type_size;
}
}
return FFIFunctionWrapperInfo(std::move(param_infos), std::move(non_fast_path_param_infos), std::move(param_map),
current_input_buf_offset, *return_info);
}
auto GenerateFFIWrapperFunction(const FunctionDecl *func_decl, llvm::raw_ostream &os) const -> llvm::Error {
auto info = ConstructFunctionWrapperInfo(func_decl);
const auto &policy = data.Ctx.getPrintingPolicy();
auto func_name = func_decl->getName();
auto wrapper_name = "__c2pnk_ffi_wrapper_" + func_name;
// add #include where func_decl is located
auto &sm = data.Ctx.getSourceManager();
auto loc = sm.getFileLoc(func_decl->getLocation());
auto file_id = sm.getFileID(loc);
std::string include_path_annotation;
if (file_id != sm.getMainFileID()) {
// sometimes external func decls can appear in the same file and only be specified during link time...
// return CreateRuntimeError(std::move(
// llvm::formatv("\n at {0}\nFFI function {1} is defined in the current source file which doesn't make
// sense",
// func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_name)));
auto file_entry = sm.getFileEntryRefForID(file_id);
if (!file_entry.has_value()) {
return CreateRuntimeError(
std::move(llvm::formatv("\n at {0}\nCouldn't get header file where FFI function {1} resides",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_name)));
}
auto &file_manager = sm.getFileManager();
auto include_path = file_manager.getCanonicalName(*file_entry);
include_path_annotation = llvm::formatv("[[clang::annotate(\"__c2pnk_ffi_include_path_{0}\")]]", include_path);
os << llvm::formatv("#include <{0}>\n", include_path);
}
os << llvm::formatv("#include <stdint.h>\n#include <string.h>\n");
for (const auto &header : function_rewrite_extra_headers) {
os << header << "\n";
}
// add #ifdef for the cursed __size_t type
// clang's qualifying type will output __size_t instead of size_t when converting to a string form
// this is broken because __size_t doesn't actually exist, only __ssize_t exists
os << "#ifndef __size_t\n"
<< " #define __size_t size_t\n"
<< "#endif\n";
const std::array<const std::string, 3> banned_prefixes = {"__builtin", "__extension", "__atomic"};
// also paste the original function's signature before the wrapper in case the original function sig is in the same
// file (and not the same file)
if (std::ranges::none_of(banned_prefixes,
[&](const std::string &prefix) -> auto { return func_name.starts_with(prefix); })) {
std::string func_decl_str;
llvm::raw_string_ostream func_decl_os(func_decl_str);
if (auto error =
PrintSourceText(func_decl_os, CharSourceRange::getTokenRange(func_decl->getSourceRange()), data.Ctx)) {
return llvm::joinErrors(CreateRuntimeError(llvm::formatv(
"\n at {0}\nCouldn't get source text for FFI function decl {1}",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_name)),
std::move(error));
}
func_decl_os << ";\n";
func_decl_os.flush();
os << func_decl_str;
}
// create the global buffers for the non-fastpath params and return value, if needed
if (auto idx = info.InputBufSlotIndex()) {
os << llvm::formatv("static uint8_t __c2pnk_ffi_input_buf_{0}[{1}];\n", func_name,
info.non_fastpath_input_buf_size);
}
if (auto idx = info.OutputBufSlotIndex()) {
if (info.return_info.IsFastPath()) {
std::string return_var_name = "__c2pnk_ffi_output_buf_" + func_name.str();
os << llvm::formatv("static {0};\n", PrintType(info.return_info.original_return_type, return_var_name));
} else {
os << llvm::formatv("static uint8_t __c2pnk_ffi_output_buf_{0}[{1}];\n", func_name,
info.return_info.size_in_bytes.value());
}
}
// emit the wrapper function signature
os << include_path_annotation << "\n";
os << "void " << wrapper_name << "(uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3) {\n";
// pull out the reserved slots, if present
if (auto idx = info.InputBufSlotIndex()) {
os << "/* input_buf = arg" << *idx << " */\n";
}
// build the argument list for the call, in original parameter order
std::string call_args;
llvm::raw_string_ostream call_args_os(call_args);
for (size_t orig_idx = 0; orig_idx < info.param_map.size(); orig_idx++) {
if (orig_idx != 0) {
call_args_os << ", ";
}
auto [fastpath_slot, non_fastpath_idx] = info.param_map[orig_idx];
if (fastpath_slot.has_value()) {
auto ¶m_info = *info.param_infos.at(*fastpath_slot);
auto param_type = param_info.original_param_decl->getType();
auto type_str = param_type.getAsString(policy);
call_args_os << "(" << type_str << ")arg" << *fastpath_slot;
} else {
auto idx = info.InputBufSlotIndex().value();
auto ¶m_info = info.non_fast_path_param_infos[*non_fastpath_idx];
auto param_type = param_info.original_param_decl->getType();
auto type_str = data.Ctx.getPointerType(param_type).getAsString(policy);
auto [offset, size] = *param_info.offset_and_size;
call_args_os << "(*(" << type_str << " )(arg" << idx << " + " << offset << "UL))";
}
}
call_args_os.flush();
// emit the call and write the return value out, choosing the cheapest path available:
// - void: just call it
// - fastpath: store the call result straight through the output pointer, no temp needed
// - non-fastpath: memcpy needs an addressable source, so a local temp is unavoidable here
if (info.return_info.original_return_type->isVoidType()) {
os << func_decl->getNameAsString() << "(" << call_args << ");\n";
} else {
auto ret_ptr_type = data.Ctx.getPointerType(info.return_info.original_return_type);
auto idx = info.OutputBufSlotIndex().value();
os << " /* output_buf = arg" << idx << " */\n";
os << "*(" << ret_ptr_type.getAsString(policy) << ")arg" << idx << " = " << func_decl->getNameAsString() << "("
<< call_args << ");\n";
}
os << "}\n";
os.flush();
return llvm::Error::success();
}
auto CallExprToFFIWrapperStmtExpr(const CallExpr *call_expr, const FunctionDecl *func_decl, llvm::raw_ostream &os)
-> llvm::Error {
auto func_name = func_decl->getName();
auto &info = data.ffi_function_info.find(func_decl)->second;
const auto &policy = data.Ctx.getPrintingPolicy();
auto wrapper_name = "__c2pnk_ffi_wrapper_" + func_decl->getName();
bool is_void_return = info.return_info.original_return_type->isVoidType();
// sometimes this can be __size_t which doesn't actually exist!!! (only __ssize_t exists)
// in this case, we need the canonical type :(
auto return_type = info.return_info.original_return_type.getCanonicalType();
// build the 4 wrapper call arguments up front
std::array<std::string, 4> slot_args = {"0UL", "0UL", "0UL", "0UL"};
for (size_t slot = 0; slot < 4; slot++) {
if (!info.param_infos.at(slot).has_value()) {
continue;
}
auto ¶m_info = *info.param_infos.at(slot);
const auto *arg_expr = call_expr->getArg(static_cast<unsigned>(param_info.original_param_index));
auto src = GetSourceText(arg_expr, data.Ctx);
if (auto error = src.takeError()) {
return llvm::joinErrors(
CreateRuntimeError(
llvm::formatv("\n at {0}\nCouldn't get source text for argument {1} of call to FFI function {2}",
call_expr->getBeginLoc().printToString(data.Ctx.getSourceManager()),
param_info.original_param_index, func_decl->getName())),
std::move(error));
}
slot_args.at(slot) = llvm::formatv("(uintptr_t)({0})", *src).str();
}
// optimization: void return + no non-fastpath params means no locals and no buffer are
// needed at all, so a plain call expression suffices
if (is_void_return && !info.HasInputSlot()) {
os << llvm::formatv("{0}({1}, {2}, {3}, {4})", wrapper_name, slot_args[0], slot_args[1], slot_args[2],
slot_args[3]);
os.flush();
return llvm::Error::success();
}
os << "({\n";
// local buffer + fill-in for whatever params didn't fit a fastpath slot
if (info.HasInputSlot()) {
for (auto ¶m_info : info.non_fast_path_param_infos) {
auto [offset, size] = *param_info.offset_and_size;
auto param_type = param_info.original_param_decl->getType();
auto index = param_info.original_param_index;
const auto *arg_i = call_expr->getArg(static_cast<unsigned>(index))->IgnoreParenImpCasts();
auto arg_i_src = GetSourceText(arg_i, data.Ctx);
if (auto error = arg_i_src.takeError()) {
return llvm::joinErrors(CreateRuntimeError(std::move(llvm::formatv(
"\n at {0}\nFailed to get source text for argument {1} of CallExpr",
arg_i->getBeginLoc().printToString(data.Ctx.getSourceManager()), index))),
std::move(error));
}
if (!func_decl->isVariadic()) {
if (!arg_i->isLValue()) {
// requires tmp var to hold the value of the argument so we can take the address of it
std::string tmp_var_name = GetTempVarName(llvm::formatv("arg_{0}", index));
os << llvm::formatv("{0} = {1};\n", PrintType(param_type, tmp_var_name), *arg_i_src);
os << llvm::formatv("__c2pnk_memcpy(__c2pnk_ffi_input_buf_{0} + {1}UL, (uint8_t *)&{2}, {3}UL);\n",
func_name, offset, tmp_var_name, size);
} else {
os << llvm::formatv("__c2pnk_memcpy(__c2pnk_ffi_input_buf_{0} + {1}UL, (uint8_t *)&{2}, {3}UL);\n",
func_name, offset, *arg_i_src, size);
}
}
}
}
if (auto idx = info.OutputBufSlotIndex()) {
slot_args.at(*idx) = llvm::formatv("(uintptr_t)&__c2pnk_ffi_output_buf_{0}", func_name);
}
if (auto idx = info.InputBufSlotIndex()) {
slot_args.at(*idx) = llvm::formatv("(uintptr_t)__c2pnk_ffi_input_buf_{0}", func_name);
}
os << llvm::formatv("{0}({1}, {2}, {3}, {4});\n", wrapper_name, slot_args[0], slot_args[1], slot_args[2],
slot_args[3]);
os.flush();
if (!is_void_return) {
auto return_type_pointer = data.Ctx.getPointerType(return_type).getAsString(policy);
os << llvm::formatv("__c2pnk_ffi_output_buf_{1};\n", return_type_pointer, func_name);
}
os << "})";
os.flush();
return llvm::Error::success();
}
auto TraverseFunctionDecl(FunctionDecl *func_decl) -> bool {
auto policy = data.Ctx.getPrintingPolicy();
auto *tmp_function_decl = data.current_function_decl;
data.current_function_decl = func_decl;
auto cleanup =
llvm::scope_exit([this, tmp_function_decl] -> void { data.current_function_decl = tmp_function_decl; });
if (!RecursiveASTVisitor<Worker>::TraverseFunctionDecl(func_decl)) {
return false;
}
if (data.error) {
return false;
}
auto &sm = data.Ctx.getSourceManager();
if (!sm.isInMainFile(sm.getSpellingLoc(func_decl->getBeginLoc()))) {
return true;
}
auto it = data.function_map.find(func_decl);
if (func_decl->isThisDeclarationADefinition()) {
if (it == data.function_map.end()) {
data.error = CreateRuntimeError(
llvm::formatv("\n at {0}\nFunctionDecl {1} not found in function_map",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName()));
return false;
}
} else {
// get def of the function decl
auto *def_decl = func_decl->getDefinition();
if (def_decl == nullptr) {
// probably an ffi function
return true;
}
it = data.function_map.find(def_decl);
if (it == data.function_map.end()) {
data.error = CreateRuntimeError(
llvm::formatv("\n at {0}\nFunctionDecl real def {1} not found in function_map",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName()));
return false;
}
}
auto function_name = func_decl->getName();
if (function_name.starts_with("__c2pnk_") || function_name == "main") {
// don't rewrite calls to pancake helper functions
return true;
}
bool is_external_entry_point = false;
for (const auto *attribute : func_decl->attrs()) {
if (const auto *annotation_attr = llvm::dyn_cast<clang::AnnotateAttr>(attribute)) {
if (annotation_attr->getAnnotation() == "__c2pnk_rewritten_non_ffi_function") {
return true;
}
if (annotation_attr->getAnnotation() == "__c2pnk_external_entry_point") {
is_external_entry_point = true;
}
}
}
auto &function_info = it->second;
if (!function_info.needs_rewriting && !function_info.turn_return_from_void_to_uint) {
return true;
}
if (opt_treat_external_functions_as_non_ffi && !func_decl->isThisDeclarationADefinition()) {
return true;
}
std::string replacement_text;
llvm::raw_string_ostream os(replacement_text);
if (func_decl->isThisDeclarationADefinition()) {
// non-ffi function def
// add global vars for hoisted return type and parameters
os << "\n/* c2pancake: start hoisted return type and parameters for function " << func_decl->getName() << " */\n";
for (auto ¶m_info : function_info.param_infos) {
if (param_info.hoisted_name.has_value()) {
os << "static " << PrintType(param_info.param_decl->getType(), param_info.hoisted_name.value()) << ";\n";
}
}
if (function_info.return_info.has_value()) {
auto &return_info = function_info.return_info.value();
if (return_info.hoisted_name.has_value()) {
os << "static " << PrintType(return_info.return_type, return_info.hoisted_name.value()) << ";\n";
}
}
os << "/* c2pancake: end hoisted return type and parameters for function " << func_decl->getName() << " */\n";
// now do the function signature
// add annotations
os << "[[clang::annotate(\"__c2pnk_rewritten_non_ffi_function\")]]\n";
for (const auto *attribute : func_decl->attrs()) {
if (const auto *annotation_attr = llvm::dyn_cast<clang::AnnotateAttr>(attribute)) {
if (annotation_attr->getAnnotation() == "__c2pnk_external_entry_point") {
// DON'T print it for this function because the name has been changed
// we instead print it for the new "wrapper" function inserted AFTER this modified function definition!
continue;
}
}
attribute->printPretty(os, data.Ctx.getPrintingPolicy());
os << "\n";
}
// function specifiers
os << GetStorageClassSpecifierString(func_decl->getStorageClass()) << " ";
if (func_decl->isInlineSpecified()) {
os << "inline ";
}
// return type
if (function_info.return_info.has_value()) {
auto &return_info = function_info.return_info.value();
if (return_info.hoisted_name.has_value()) {
// the return var is actually passed through a global var
// but the function ret types still need to be a uint32/64_t as per pancake rules
os << GetWordTypeStr(data.Ctx) << " ";
} else {
os << return_info.return_type.getAsString(policy) << " ";
}
} else {
// void return type becomes a uint32/64_t as per pancake rules
os << GetWordTypeStr(data.Ctx) << " ";
}
// name
if (is_external_entry_point) {
os << "__c2pnk_external_entry_point_";
}
os << function_name;
// args
os << "(";
for (const auto ¶m_info : function_info.param_infos) {
if (!param_info.hoisted_name.has_value()) {
os << PrintType(param_info.param_decl->getType(), param_info.param_decl->getName()) << ", ";
}
}
os.flush();
if (replacement_text.ends_with(", ")) {
replacement_text.resize(replacement_text.size() - 2);
}
os << ")";
// now do the function prologue to load all the hoisted parameters into local variables
os << " {\n";
os << "/* c2pancake: start function prologue for function " << function_name << " */\n";
for (const auto ¶m_info : function_info.param_infos) {
if (param_info.hoisted_name.has_value()) {
os << llvm::formatv("{0} = {1};\n",
PrintType(param_info.param_decl->getType(), param_info.param_decl->getName()),
param_info.hoisted_name.value());
}
}
os << "/* c2pancake: end function prologue for function " << function_name << " */\n";
os.flush();
// replace only "ret_type func_name(param_type1 param1, param_type2 param2, ...) {"
// note begin loc DOESn't include the attributes so the external entry point annotation will be duplicated.
// the real way is to use the name of the func to decide during the final pass
auto begin_loc = func_decl->getBeginLoc();
auto l_brace = func_decl->getBody()->getBeginLoc(); // points at '{'
auto after_l_brace = Lexer::getLocForEndOfToken(l_brace, 0, sm, data.Ctx.getLangOpts());
auto range = CharSourceRange::getCharRange(begin_loc, after_l_brace);
if (auto error =
data.non_ffi_rewrites[func_decl].add(Replacement(sm, range, replacement_text, data.Ctx.getLangOpts()))) {
data.error = CreateRuntimeError(
llvm::formatv("\n at {0}\nNon-ffi function definition rewrite for function {1} FAILED",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName()));
return false;
}
os.flush();
if (is_external_entry_point) {
// we renamed the function earlier to prepend __c2pnk_external_entry_point_ to the name
//
// Now we want to add a new function with the original name that calls the renamed function.
std::string new_func_text;
llvm::raw_string_ostream new_func_os(new_func_text);
// add annotations
new_func_os << "\n\n[[clang::annotate(\"__c2pnk_external_entry_point\")]]\n";
new_func_os << "[[clang::annotate(\"__c2pnk_rewritten_non_ffi_function\")]]\n";
// grab the signature part of the original definition
auto before_l_brace = l_brace.getLocWithOffset(-1);
auto func_sig =
Lexer::getSourceText(CharSourceRange::getCharRange(begin_loc, before_l_brace), sm, data.Ctx.getLangOpts());
new_func_os << func_sig << " {\n";
// add a call to the renamed function
std::string call_text;
llvm::raw_string_ostream call_os(call_text);
call_os << "__c2pnk_external_entry_point_" << function_name << "(";
// forward the args
for (const auto ¶m_info : function_info.param_infos) {
if (param_info.hoisted_name.has_value()) {
// this is funadmentally incompatible!
data.error = CreateRuntimeError(
std::move(llvm::formatv("\n at {0}\nExternal entry point function {1} needs a hoisted parameter "
"{2}, which is not supported",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()),
func_decl->getName(), param_info.param_decl->getName())));
return false;
}
if (!param_info.param_decl->getType()->isVoidType()) {
call_os << param_info.param_decl->getName() << ", ";
}
}
call_os.flush();
if (call_text.ends_with(", ")) {
call_text.resize(call_text.size() - 2);
}
call_os << ")";
call_os.flush();
if (function_info.turn_return_from_void_to_uint) {
// don't add a return value since the original function returns void
new_func_os << call_text << ";\n";
} else {
new_func_os << " return " << call_text << ";\n";
}
// end the function
new_func_os << "}\n";
new_func_os.flush();
// add the new function AFTER the original function definition
auto after_r_brace =
Lexer::getLocForEndOfToken(func_decl->getBody()->getEndLoc(), 0, sm, data.Ctx.getLangOpts());
// bit of a hack putting it in the return rewrites...
if (auto error = data.non_ffi_rewrites[func_decl].add(Replacement(sm, after_r_brace, 0, new_func_text))) {
data.error = llvm::joinErrors(
CreateRuntimeError(std::move(llvm::formatv(
"\n at {0}\nFailed to add non-ffi function definition rewrite for function {1}",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName()))),
std::move(error));
return false;
}
}
if (function_info.turn_return_from_void_to_uint) {
// insert a return 0UL at the end of the function right before the closing brace
auto r_brace = func_decl->getBody()->getEndLoc(); // location of '}'
auto range = CharSourceRange::getCharRange(r_brace, r_brace);
if (auto error =
data.non_ffi_rewrites[func_decl].add(Replacement(sm, range, "return 0UL;\n", data.Ctx.getLangOpts()))) {
data.error = CreateRuntimeError(
llvm::formatv("\n at {0}\nNon-ffi function definition {1} already has a return rewrite",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName()));
return false;
}
}
} else {
// non-ffi function decl
// don't touch the function signature at all.
if (is_external_entry_point) {
return RecursiveASTVisitor<Worker>::TraverseFunctionDecl(func_decl);
}
// attributes
for (const auto *attribute : func_decl->attrs()) {
attribute->printPretty(os, data.Ctx.getPrintingPolicy());
os << "\n";
}
// function specifiers
os << GetStorageClassSpecifierString(func_decl->getStorageClass()) << " ";
if (func_decl->isInlineSpecified()) {
os << "inline ";
}
// return type
if (function_info.return_info.has_value()) {
auto &return_info = function_info.return_info.value();
if (return_info.hoisted_name.has_value()) {
// the return var is actually passed through a global var
// but the function ret types still need to be a uint32/64_t as per pancake rules
os << GetWordTypeStr(data.Ctx) << " ";
} else {
os << return_info.return_type.getAsString(policy) << " ";
}
} else {
// void return type becomes a uint32/64_t as per pancake rules
os << GetWordTypeStr(data.Ctx) << " ";
}
// function name
os << function_name << "(";
// parameters
for (const auto ¶m_info : function_info.param_infos) {
if (!param_info.hoisted_name.has_value()) {
os << PrintType(param_info.param_decl->getType(), param_info.param_decl->getName()) << ", ";
}
}
os.flush();
if (replacement_text.ends_with(", ")) {
replacement_text.resize(replacement_text.size() - 2);
}
os << ")";
os.flush();
// add the replacement under the func definition's decl!
if (auto error = data.non_ffi_rewrites[func_decl->getDefinition()].add(
{data.Ctx.getSourceManager(), CharSourceRange::getTokenRange(func_decl->getSourceRange()),
replacement_text, data.Ctx.getLangOpts()})) {
data.error = llvm::joinErrors(
CreateRuntimeError(std::move(llvm::formatv(
"\n at {0}\nFailed to add non-ffi function declaration rewrite for function {1}",
func_decl->getBeginLoc().printToString(data.Ctx.getSourceManager()), func_decl->getName()))),
std::move(error));
return false;
};
}
return true;
}
auto TraverseCallExpr(CallExpr *call_expr) -> bool {
if (!RecursiveASTVisitor<Worker>::TraverseCallExpr(call_expr)) {
return false;
}
if (data.error) {
return false;
}
auto &sm = data.Ctx.getSourceManager();
if (!sm.isInMainFile(sm.getSpellingLoc(call_expr->getBeginLoc()))) {
return true;
}
auto *callee_decl = call_expr->getDirectCallee();
if (callee_decl == nullptr) {
// maybe function pointer???
data.error = CreateRuntimeError(
std::move(llvm::formatv("\n at {0}\nCallExpr has no direct callee",
call_expr->getBeginLoc().printToString(data.Ctx.getSourceManager()))));
return false;
}