Halide 18.0.0
Halide compiler and libraries
Loading...
Searching...
No Matches
simd_op_check.h
Go to the documentation of this file.
1#ifndef SIMD_OP_CHECK_H
2#define SIMD_OP_CHECK_H
3
4#include "Halide.h"
5#include "halide_test_dirs.h"
6#include "halide_thread_pool.h"
7#include "test_sharding.h"
8
9#include <fstream>
10#include <iostream>
11
12namespace {
13
14using namespace Halide;
15
16// Some exprs of each type to use in checked expressions. These will turn
17// into loads to thread-local image params.
18Expr input(const Type &t, const Expr &arg) {
19 return Internal::Call::make(t, "input", {arg}, Internal::Call::Extern);
20}
21Expr in_f16(const Expr &arg) {
22 return input(Float(16), arg);
23}
24Expr in_bf16(const Expr &arg) {
25 return input(BFloat(16), arg);
26}
27Expr in_f32(const Expr &arg) {
28 return input(Float(32), arg);
29}
30Expr in_f64(const Expr &arg) {
31 return input(Float(64), arg);
32}
33Expr in_i8(const Expr &arg) {
34 return input(Int(8), arg);
35}
36Expr in_i16(const Expr &arg) {
37 return input(Int(16), arg);
38}
39Expr in_i32(const Expr &arg) {
40 return input(Int(32), arg);
41}
42Expr in_i64(const Expr &arg) {
43 return input(Int(64), arg);
44}
45Expr in_u8(const Expr &arg) {
46 return input(UInt(8), arg);
47}
48Expr in_u16(const Expr &arg) {
49 return input(UInt(16), arg);
50}
51Expr in_u32(const Expr &arg) {
52 return input(UInt(32), arg);
53}
54Expr in_u64(const Expr &arg) {
55 return input(UInt(64), arg);
56}
57} // namespace
58
59namespace Halide {
60struct TestResult {
61 std::string op;
62 std::string error_msg;
63};
64
65struct Task {
66 std::string op;
67 std::string name;
70};
71
73public:
74 static constexpr int max_i8 = 127;
75 static constexpr int max_i16 = 32767;
76 static constexpr int max_i32 = 0x7fffffff;
77 static constexpr int max_u8 = 255;
78 static constexpr int max_u16 = 65535;
79 const Expr max_u32 = UInt(32).max();
80
81 std::string filter{"*"};
83 std::vector<Task> tasks;
84
86
87 int W;
88 int H;
89
91
93
101 virtual ~SimdOpCheckTest() = default;
102
103 void set_seed(int seed) {
104 rng_seed = seed;
105 }
106
107 virtual bool can_run_code() const {
110 }
111 // If we can (target matches host), run the error checking Halide::Func.
113 bool can_run_the_code =
114 (target.arch == host_target.arch &&
115 target.bits == host_target.bits &&
116 target.os == host_target.os);
117 // A bunch of feature flags also need to match between the
118 // compiled code and the host in order to run the code.
119 for (Target::Feature f : {
141 }) {
142 if (target.has_feature(f) != host_target.has_feature(f)) {
143 can_run_the_code = false;
144 }
145 }
146 return can_run_the_code;
147 }
148
149 virtual void compile_and_check(Func error,
150 const std::string &op,
151 const std::string &name,
152 int vector_width,
153 const std::vector<Argument> &arg_types,
154 std::ostringstream &error_msg) {
155 std::string fn_name = "test_" + name;
156 std::string file_name = output_directory + fn_name;
157
159 std::map<OutputFileType, std::string> outputs = {
163 };
164 error.compile_to(outputs, arg_types, fn_name, target);
165
166 std::ifstream asm_file;
167 asm_file.open(file_name + ".s");
168
169 bool found_it = false;
170
171 std::ostringstream msg;
172 msg << op << " did not generate for target=" << get_run_target().to_string() << " vector_width=" << vector_width << ". Instead we got:\n";
173
174 std::string line;
175 while (getline(asm_file, line)) {
176 msg << line << "\n";
177
178 // Check for the op in question
179 found_it |= wildcard_search(op, line) && !wildcard_search("_" + op, line);
180 }
181
182 if (!found_it) {
183 error_msg << "Failed: " << msg.str() << "\n";
184 }
185
186 asm_file.close();
187 }
188
189 // Check if pattern p matches str, allowing for wildcards (*).
190 bool wildcard_match(const char *p, const char *str) const {
191 // Match all non-wildcard characters.
192 while (*p && *str && *p == *str && *p != '*') {
193 str++;
194 p++;
195 }
196
197 if (!*p) {
198 return *str == 0;
199 } else if (*p == '*') {
200 p++;
201 do {
202 if (wildcard_match(p, str)) {
203 return true;
204 }
205 } while (*str++);
206 } else if (*p == ' ') { // ignore whitespace in pattern
207 p++;
208 if (wildcard_match(p, str)) {
209 return true;
210 }
211 } else if (*str == ' ') { // ignore whitespace in string
212 str++;
213 if (wildcard_match(p, str)) {
214 return true;
215 }
216 }
217 return !*p;
218 }
219
220 bool wildcard_match(const std::string &p, const std::string &str) const {
221 return wildcard_match(p.c_str(), str.c_str());
222 }
223
224 // Check if a substring of str matches a pattern p.
225 bool wildcard_search(const std::string &p, const std::string &str) const {
226 return wildcard_match("*" + p + "*", str);
227 }
228
235
236 TestResult check_one(const std::string &op, const std::string &name, int vector_width, Expr e) {
237 std::ostringstream error_msg;
238
239 // Map the input calls in the Expr to loads to local
240 // imageparams, so that we're not sharing state across threads.
241 std::vector<ImageParam> image_params{
242 ImageParam{Float(32), 1, "in_f32"},
243 ImageParam{Float(64), 1, "in_f64"},
244 ImageParam{Float(16), 1, "in_f16"},
245 ImageParam{BFloat(16), 1, "in_bf16"},
246 ImageParam{Int(8), 1, "in_i8"},
247 ImageParam{UInt(8), 1, "in_u8"},
248 ImageParam{Int(16), 1, "in_i16"},
249 ImageParam{UInt(16), 1, "in_u16"},
250 ImageParam{Int(32), 1, "in_i32"},
251 ImageParam{UInt(32), 1, "in_u32"},
252 ImageParam{Int(64), 1, "in_i64"},
253 ImageParam{UInt(64), 1, "in_u64"}};
254
255 for (auto &p : image_params) {
257 p.set_host_alignment(alignment_bytes);
258 const int alignment = alignment_bytes / p.type().bytes();
259 p.dim(0).set_min((p.dim(0).min() / alignment) * alignment);
260 }
261
262 const std::vector<Argument> arg_types(image_params.begin(), image_params.end());
263
266
267 Expr visit(const Internal::Call *op) override {
268 if (op->name == "input") {
269 for (auto &p : image_params) {
270 if (p.type() == op->type) {
271 return p(mutate(op->args[0]));
272 }
273 }
274 } else if (op->call_type == Internal::Call::Halide && !op->func.weak) {
276 f.mutate(this);
277 }
279 }
280 const std::vector<ImageParam> &image_params;
281
282 public:
283 HookUpImageParams(const std::vector<ImageParam> &image_params)
285 }
287 e = hook_up_image_params.mutate(e);
288
291 void visit(const Internal::Call *op) override {
294 if (f.has_update_definition() &&
295 f.update(0).schedule().rvars().size() > 0) {
297 result = true;
298 }
299 }
300 IRVisitor::visit(op);
301 }
302
303 public:
305 bool result = false;
308
309 // Define a vectorized Halide::Func that uses the pattern.
310 Halide::Func f(name);
311 f(x, y) = e;
312 f.bound(x, 0, W).vectorize(x, vector_width);
313 f.compute_root();
314
315 // Include a scalar version
316 Halide::Func f_scalar("scalar_" + name);
317 f_scalar(x, y) = e;
318
319 if (has_inline_reduction.result) {
320 // If there's an inline reduction, we want to vectorize it
321 // over the RVar.
322 Var xo, xi;
323 RVar rxi;
324 Func g{has_inline_reduction.inline_reduction};
325
326 // Do the reduction separately in f_scalar
327 g.clone_in(f_scalar);
328
329 g.compute_at(f, x)
330 .update()
331 .split(x, xo, xi, vector_width)
332 .atomic(true)
333 .vectorize(g.rvars()[0])
334 .vectorize(xi);
335 }
336
337 // The output to the pipeline is the maximum absolute difference as a double.
338 RDom r_check(0, W, 0, H);
339 Halide::Func error("error_" + name);
341
342 compile_and_check(error, op, name, vector_width, arg_types, error_msg);
343
345 if (can_run_the_code) {
347
348 // Make some unallocated input buffers
349 std::vector<Runtime::Buffer<>> inputs(image_params.size());
350
351 std::vector<Argument> args(image_params.size());
352 for (size_t i = 0; i < args.size(); i++) {
353 args[i] = image_params[i];
354 inputs[i] = Runtime::Buffer<>(args[i].type, nullptr, 0);
355 }
356 auto callable = error.compile_to_callable(args, run_target);
357
359 output(0) = 1; // To ensure we'll fail if it's never written to
360
361 // Do the bounds query call
362 assert(inputs.size() == 12);
363 (void)callable(inputs[0], inputs[1], inputs[2], inputs[3],
364 inputs[4], inputs[5], inputs[6], inputs[7],
365 inputs[8], inputs[9], inputs[10], inputs[11],
366 output);
367
368 std::mt19937 rng;
369 rng.seed(rng_seed);
370
371 // Allocate the input buffers and fill them with noise
372 for (size_t i = 0; i < inputs.size(); i++) {
373 if (inputs[i].size_in_bytes()) {
374 inputs[i].allocate();
375
376 Type t = inputs[i].type();
377 // For floats/doubles, we only use values that aren't
378 // subject to rounding error that may differ between
379 // vectorized and non-vectorized versions
380 if (t == Float(32)) {
381 inputs[i].as<float>().for_each_value([&](float &f) { f = (rng() & 0xfff) / 8.0f - 0xff; });
382 } else if (t == Float(64)) {
383 inputs[i].as<double>().for_each_value([&](double &f) { f = (rng() & 0xfff) / 8.0 - 0xff; });
384 } else if (t == Float(16)) {
385 inputs[i].as<float16_t>().for_each_value([&](float16_t &f) { f = float16_t((rng() & 0xff) / 8.0f - 0xf); });
386 } else {
387 // Random bits is fine
388 for (uint32_t *ptr = (uint32_t *)inputs[i].data();
389 ptr != (uint32_t *)inputs[i].data() + inputs[i].size_in_bytes() / 4;
390 ptr++) {
391 // Never use the top four bits, to avoid
392 // signed integer overflow.
393 *ptr = ((uint32_t)rng()) & 0x0fffffff;
394 }
395 }
396 }
397 }
398
399 // Do the real call
400 (void)callable(inputs[0], inputs[1], inputs[2], inputs[3],
401 inputs[4], inputs[5], inputs[6], inputs[7],
402 inputs[8], inputs[9], inputs[10], inputs[11],
403 output);
404
405 double e = output(0);
406 // Use a very loose tolerance for floating point tests. The
407 // kinds of bugs we're looking for are codegen bugs that
408 // return the wrong value entirely, not floating point
409 // accuracy differences between vectors and scalars.
410 if (e > 0.001) {
411 error_msg << "The vector and scalar versions of " << name << " disagree. Maximum error: " << e << "\n";
412
413 std::string error_filename = output_directory + "error_" + name + ".s";
414 error.compile_to_assembly(error_filename, arg_types, target);
415
416 std::ifstream error_file;
418
419 error_msg << "Error assembly: \n";
420 std::string line;
421 while (getline(error_file, line)) {
422 error_msg << line << "\n";
423 }
424
425 error_file.close();
426 }
427 }
428
429 return {op, error_msg.str()};
430 }
431
432 void check(std::string op, int vector_width, Expr e) {
433 // Make a name for the test by uniquing then sanitizing the op name
434 std::string name = "op_" + op;
435 for (size_t i = 0; i < name.size(); i++) {
436 if (!isalnum(name[i])) name[i] = '_';
437 }
438
439 name += "_" + std::to_string(tasks.size());
440
441 // Bail out after generating the unique_name, so that names are
442 // unique across different processes and don't depend on filter
443 // settings.
444 if (!wildcard_match(filter, op)) return;
445
446 tasks.emplace_back(Task{op, name, vector_width, e});
447 }
448 virtual void add_tests() = 0;
449 virtual int image_param_alignment() {
450 return 16;
451 }
452
453 virtual bool use_multiple_threads() const {
454 return true;
455 }
456
457 virtual bool test_all() {
458 /* First add some tests based on the target */
459 add_tests();
460
461 // Remove irrelevant noise from output
463 const std::string run_target_str = run_target.to_string();
464
466
467 Halide::Tools::ThreadPool<TestResult> pool(
469 Halide::Tools::ThreadPool<TestResult>::num_processors_online() :
470 1);
471 std::vector<std::future<TestResult>> futures;
472
473 for (size_t t = 0; t < tasks.size(); t++) {
474 if (!sharder.should_run(t)) continue;
475 const auto &task = tasks.at(t);
476 futures.push_back(pool.async([&]() {
477 return check_one(task.op, task.name, task.vector_width, task.expr);
478 }));
479 }
480
481 for (auto &f : futures) {
482 auto result = f.get();
483 constexpr int tabstop = 32;
484 const int spaces = std::max(1, tabstop - (int)result.op.size());
485 std::cout << result.op << std::string(spaces, ' ') << "(" << run_target_str << ")\n";
486 if (!result.error_msg.empty()) {
487 std::cerr << result.error_msg;
488 // The thread-pool destructor will block until in-progress tasks
489 // are done, and then will discard any tasks that haven't been
490 // launched yet.
491 return false;
492 }
493 }
494
495 return true;
496 }
497
498 template<typename SIMDOpCheckT>
499 static int main(int argc, char **argv, const std::vector<Target> &targets_to_test) {
500 Target host = get_host_target();
501 std::cout << "host is: " << host << "\n";
502
503 const int seed = argc > 2 ? atoi(argv[2]) : time(nullptr);
504 std::cout << "simd_op_check test seed: " << seed << "\n";
505
506 for (const auto &t : targets_to_test) {
507 if (!t.supported()) {
508 std::cout << "[SKIP] Unsupported target: " << t << "\n";
509 return 0;
510 }
511 SIMDOpCheckT test(t);
512
513 if (!t.supported()) {
514 std::cout << "Halide was compiled without support for " << t.to_string() << ". Skipping.\n";
515 continue;
516 }
517
518 if (argc > 1) {
519 test.filter = argv[1];
520 }
521
522 if (getenv("HL_SIMD_OP_CHECK_FILTER")) {
523 test.filter = getenv("HL_SIMD_OP_CHECK_FILTER");
524 }
525
526 test.set_seed(seed);
527
528 if (argc > 2) {
529 // Don't forget: if you want to run the standard tests to a specific output
530 // directory, you'll need to invoke with the first arg enclosed
531 // in quotes (to avoid it being wildcard-expanded by the shell):
532 //
533 // correctness_simd_op_check "*" /path/to/output
534 //
535 test.output_directory = argv[2];
536 }
537
538 bool success = test.test_all();
539
540 // Compile a runtime for this target, for use in the static test.
541 compile_standalone_runtime(test.output_directory + "simd_op_check_runtime.o", test.target);
542
543 if (!success) {
544 return 1;
545 }
546 }
547
548 std::cout << "Success!\n";
549 return 0;
550 }
551
552private:
553 const Halide::Var x{"x"}, y{"y"};
554};
555
556} // namespace Halide
557
558#endif // SIMD_OP_CHECK_H
A halide function.
Definition Func.h:700
void compile_to_assembly(const std::string &filename, const std::vector< Argument > &, const std::string &fn_name, const Target &target=get_target_from_environment())
Statically compile this function to text assembly equivalent to the object file generated by compile_...
Func & compute_root()
Compute all of this function once ahead of time.
Callable compile_to_callable(const std::vector< Argument > &args, const Target &target=get_jit_target_from_environment())
Eagerly jit compile the function to machine code and return a callable struct that behaves like a fun...
void compile_to(const std::map< OutputFileType, std::string > &output_files, const std::vector< Argument > &args, const std::string &fn_name, const Target &target=get_target_from_environment())
Compile and generate multiple target files with single call.
Func & vectorize(const VarOrRVar &var)
Mark a dimension to be computed all-at-once as a single vector.
Func & bound(const Var &var, Expr min, Expr extent)
Statically declare that the range over which a function should be evaluated is given by the second an...
An Image parameter to a halide pipeline.
Definition ImageParam.h:23
const StageSchedule & schedule() const
Get the default (no-specialization) stage-specific schedule associated with this definition.
A reference-counted handle to Halide's internal representation of a function.
Definition Function.h:39
bool has_update_definition() const
Does this function have an update definition?
void mutate(IRMutator *mutator)
Accept a mutator to mutator all of the definitions and arguments of this function.
Definition & update(int idx=0)
Get a mutable handle to this function's update definition at index 'idx'.
A base class for passes over the IR which modify it (e.g.
Definition IRMutator.h:26
virtual Expr visit(const IntImm *)
A base class for algorithms that need to recursively walk over the IR.
Definition IRVisitor.h:19
virtual void visit(const IntImm *)
const std::vector< ReductionVariable > & rvars() const
RVars of reduction domain associated with this schedule if there is any.
A multi-dimensional domain over which to iterate.
Definition RDom.h:193
A reduction variable represents a single dimension of a reduction domain (RDom).
Definition RDom.h:29
A templated Buffer class that wraps halide_buffer_t and adds functionality.
static Buffer< T, Dims, InClassDimStorage > make_scalar()
Make a zero-dimensional Buffer.
static constexpr int max_u8
static constexpr int max_i32
virtual bool use_multiple_threads() const
virtual void add_tests()=0
virtual int image_param_alignment()
bool wildcard_match(const std::string &p, const std::string &str) const
static constexpr int max_i8
static constexpr int max_u16
bool wildcard_search(const std::string &p, const std::string &str) const
bool wildcard_match(const char *p, const char *str) const
virtual ~SimdOpCheckTest()=default
SimdOpCheckTest(const Target t, int w, int h)
void check(std::string op, int vector_width, Expr e)
Target get_run_target() const
static int main(int argc, char **argv, const std::vector< Target > &targets_to_test)
static constexpr int max_i16
TestResult check_one(const std::string &op, const std::string &name, int vector_width, Expr e)
virtual bool can_run_code() const
std::vector< Task > tasks
virtual void compile_and_check(Func error, const std::string &op, const std::string &name, int vector_width, const std::vector< Argument > &arg_types, std::ostringstream &error_msg)
A Halide variable, to be used when defining functions.
Definition Var.h:19
std::map< OutputFileType, const OutputInfo > get_output_info(const Target &target)
std::string get_test_tmp_dir()
Return the path to a directory that can be safely written to when running tests; the contents directo...
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Target get_host_target()
Return the target corresponding to the host machine.
Type BFloat(int bits, int lanes=1)
Construct a floating-point type in the bfloat format.
Definition Type.h:556
Type UInt(int bits, int lanes=1)
Constructing an unsigned integer type.
Definition Type.h:546
Type Float(int bits, int lanes=1)
Construct a floating-point type.
Definition Type.h:551
Expr maximum(Expr, const std::string &s="maximum")
Type Int(int bits, int lanes=1)
Constructing a signed integer type.
Definition Type.h:541
Expr absd(Expr a, Expr b)
Return the absolute difference between two values.
void compile_standalone_runtime(const std::string &object_filename, const Target &t)
Create an object file containing the Halide runtime for a given target.
Internal::ConstantInterval cast(Type t, const Internal::ConstantInterval &a)
Cast operators for ConstantIntervals.
int atoi(const char *)
unsigned __INT32_TYPE__ uint32_t
char * getenv(const char *)
A fragment of Halide syntax.
Definition Expr.h:258
A function call.
Definition IR.h:490
@ Extern
A call to an external C-ABI function, possibly with side-effects.
Definition IR.h:494
@ Halide
A call to a Func.
Definition IR.h:497
std::string name
Definition IR.h:491
FunctionPtr func
Definition IR.h:667
CallType call_type
Definition IR.h:501
std::vector< Expr > args
Definition IR.h:492
static Expr make(Type type, IntrinsicOp op, const std::vector< Expr > &args, CallType call_type, FunctionPtr func=FunctionPtr(), int value_index=0, const Buffer<> &image=Buffer<>(), Parameter param=Parameter())
int64_t min
The lower and upper bound of the interval.
void accept(IRVisitor *v) const
Dispatch to the correct visitor method for this node.
Definition Expr.h:192
static bool can_jit_target(const Target &target)
If the given target can be executed via the wasm executor, return true.
A struct representing a target machine and os to generate code for.
Definition Target.h:19
enum Halide::Target::Arch arch
bool has_feature(Feature f) const
int bits
The bit-width of the target machine.
Definition Target.h:50
enum Halide::Target::OS os
std::string to_string() const
Convert the Target into a string form that can be reconstituted by merge_string(),...
Target without_feature(Feature f) const
Return a copy of the target with the given feature cleared.
Feature
Optional features a target can have.
Definition Target.h:83
@ AVX512_Cannonlake
Definition Target.h:132
@ AVX512_SapphireRapids
Definition Target.h:133
@ POWER_ARCH_2_07
Definition Target.h:97
Target with_feature(Feature f) const
Return a copy of the target with the given feature set.
std::string op
std::string name
std::string error_msg
Types in the halide type system.
Definition Type.h:283
Expr max() const
Return an expression which is the maximum value of this type.
Class that provides a type that implements half precision floating point (IEEE754 2008 binary16) in s...
Definition Float16.h:17