Skip to content

Commit 7c404c4

Browse files
KDr2KDr2
authored
use memory layout info to copy task (#89)
* use memory layout info to copy task * offset selection * upgrade Libtask_jll to v0.5.1 * support linux-x86 * correct ARCH name * layout info on macOS * run test on each platform regardless of the results on others * win64 support * win32 support * memlayout workflow * Commit from GitHub Actions (MemLayout Info Generating) * Commit from GitHub Actions (MemLayout Info Generating) * only generate layout file on PR * rename exe file * Commit from GitHub Actions (MemLayout Info Generating) * include mem layout info of v1.5.3 * move memlayout generator to a subfolder
1 parent 17b12a0 commit 7c404c4

35 files changed

+1103
-5
lines changed

‎.github/workflows/MemLayoutGen.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: MemLayout Info Generating
2+
on:
3+
pull_request:
4+
jobs:
5+
test:
6+
runs-on: ${{ matrix.os }}
7+
continue-on-error: true
8+
strategy:
9+
matrix:
10+
version:
11+
- '1.3.1'
12+
- '1.4.2'
13+
- '1.5.3'
14+
- '1.5.4'
15+
- '1.6.1'
16+
os:
17+
- ubuntu-latest
18+
- windows-latest
19+
- macOS-latest
20+
arch:
21+
- x64
22+
- x86
23+
exclude:
24+
- os: macOS-latest
25+
arch: x86
26+
steps:
27+
- name: Set up MinGW
28+
uses: egor-tensin/setup-mingw@v2
29+
with:
30+
platform: i686
31+
cc: 1
32+
if: ${{ matrix.os == 'windows-latest' && matrix.arch == 'x86' }}
33+
- uses: actions/checkout@v2
34+
- uses: julia-actions/setup-julia@v1
35+
with:
36+
version: ${{ matrix.version }}
37+
arch: ${{ matrix.arch }}
38+
- run: julia .github/workflows/tasklayout/memlayout.jl
39+
- uses: EndBug/add-and-commit@v7
40+
with:
41+
add: 'src/memlayout'
42+
author_name: GitHub Actions

‎.github/workflows/Testing.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ on:
77
jobs:
88
test:
99
runs-on: ${{ matrix.os }}
10-
continue-on-error: ${{ matrix.version == 'nightly' }}
10+
continue-on-error: true # ${{ matrix.version == 'nightly' }}
1111
strategy:
1212
matrix:
1313
version:

‎.github/workflows/tasklayout/memlayout.cpp‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/**
2+
* linux64: g++ -I $JULIA_HOME/include/julia/ memlayout.cpp -o memlayout
3+
* linux32: apt-get install g++-multilib
4+
* g++ -march=pentium4 -m32 -I $JULIA_HOME/include/julia/ memlayout.cpp -o memlayout
5+
* macOS: g++ -std=c++11 -I $JULIA_APP/Contents/Resources/julia/include/julia/ memlayout.cpp -o memlayout
6+
* win64: g++ -I $JULIA_HOME/include/julia/ memlayout.cpp -o memlayout
7+
* win32: g++ -I $JULIA_HOME/include/julia/ memlayout.cpp -o memlayout
8+
**/
9+
10+
#include <string>
11+
#include <map>
12+
#include <fstream>
13+
#include <sstream>
14+
15+
#include <cstddef>
16+
17+
#include "julia.h"
18+
19+
std::string jl_ver(std::string sep=".") {
20+
std::ostringstream oss;
21+
oss << JULIA_VERSION_MAJOR << sep
22+
<< JULIA_VERSION_MINOR << sep
23+
<< JULIA_VERSION_PATCH;
24+
return oss.str();
25+
}
26+
27+
std::string platform() {
28+
#ifdef _WIN32
29+
#ifdef _WIN64
30+
return "windows-x86_64";
31+
#endif
32+
return "windows-x86";
33+
#endif
34+
#ifdef __APPLE__
35+
return "darwin-x86_64";
36+
#endif
37+
#ifdef __linux__
38+
#ifdef __x86_64
39+
return "linux-x86_64";
40+
#else
41+
return "linux-x86";
42+
#endif
43+
#endif
44+
}
45+
46+
std::map<std::string, size_t> task_field_offsets() {
47+
std::map<std::string, size_t> data;
48+
49+
#define field_info(f) data[#f] = offsetof(jl_task_t, f)
50+
51+
field_info(next); // 131 142 153 154 160 170
52+
field_info(queue); // 131 142 153 154 160 170
53+
field_info(tls); // 131 142 153 154 160 170
54+
field_info(donenotify); // 131 142 153 154 160 170
55+
field_info(result); // 131 142 153 154 160 170
56+
57+
#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR < 6
58+
field_info(state); // 131 142 153 154
59+
field_info(exception); // 131 142 153 154
60+
field_info(backtrace); // 131 142 153 154
61+
#endif
62+
63+
field_info(logstate); // 131 142 153 154 160 170
64+
field_info(start); // 131 142 153 154 160 170
65+
field_info(sticky); // 131 142 153 154 160 170
66+
67+
#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR > 5
68+
field_info(_state); // 160 170
69+
field_info(_isexception); // 160 170
70+
#endif
71+
72+
#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR > 6
73+
field_info(rngState0); // 170
74+
field_info(rngState1); // 170
75+
field_info(rngState2); // 170
76+
field_info(rngState3); // 170
77+
#endif
78+
79+
// hidden state
80+
field_info(gcstack); // 131 142 153 154 160 170
81+
82+
#if JULIA_VERSION_MAJOR == 1 && \
83+
(JULIA_VERSION_MINOR > 6 || JULIA_VERSION_MINOR < 5 || \
84+
(JULIA_VERSION_MINOR == 5 && JULIA_VERSION_PATCH < 4))
85+
field_info(world_age); // 131 142 153 170
86+
#endif
87+
88+
#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR > 6
89+
field_info(ptls); // 170
90+
#endif
91+
field_info(tid); // 131 142 153 154 160 170
92+
field_info(prio); // 131 142 153 154 160 170
93+
field_info(excstack); // 131 142 153 154 160 170
94+
field_info(eh); // 131 142 153 154 160 170
95+
field_info(ctx); // 131 142 153 154 160 170
96+
97+
#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR < 6
98+
field_info(locks); // 131 142 153 154
99+
field_info(timing_stack); // 131 142 153 154
100+
#endif
101+
// field_info(copy_stack_ctx);
102+
103+
#if defined(JL_TSAN_ENABLED)
104+
field_info(tsan_state); // 170
105+
#endif
106+
107+
field_info(stkbuf); // 131 142 153 154 160 170
108+
field_info(bufsz); // 131 142 153 154 160 170
109+
// field_info(copy_stack);
110+
// field_info(started);
111+
112+
#undef field_info
113+
data["copy_stack"] = offsetof(jl_task_t, bufsz) + sizeof(size_t);
114+
data["sizeof_ctx"] = sizeof(((jl_task_t*)0)->ctx);
115+
// data["sizeof_stack_ctx"] = sizeof(((jl_task_t*)0)->copy_stack_ctx);
116+
data["tls_base_context"] = offsetof(struct _jl_tls_states_t, base_ctx);
117+
// data["tls_copy_stack_ctx"] = offsetof(struct _jl_tls_states_t, copy_stack_ctx);
118+
return data;
119+
}
120+
121+
int main() {
122+
std::ofstream ofs(platform() +"-v" + jl_ver("_") + ".jl", std::ofstream::out);
123+
124+
ofs << "ALL_TASK_OFFSETS[("
125+
<< '"' << platform() << '"' << ", "
126+
<< "v\"" << jl_ver() <<"\""
127+
<<")] = Dict(\n";
128+
ofs << " :END => " << sizeof(jl_task_t) << ",\n";
129+
130+
std::map<std::string, size_t> data = task_field_offsets();
131+
for(auto kv: data) {
132+
ofs << " :" << kv.first << " => " << kv.second << ",\n";
133+
}
134+
ofs << ")\n";
135+
136+
ofs.close();
137+
return 0;
138+
}

‎.github/workflows/tasklayout/memlayout.jl‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
const ARCH = @static if string(Sys.ARCH)[1] == 'i'
2+
"x86"
3+
else
4+
string(Sys.ARCH)
5+
end
6+
7+
const PLATFORM = @static if Sys.islinux()
8+
"linux-" * ARCH
9+
elseif Sys.iswindows()
10+
"windows-" * ARCH
11+
elseif Sys.isapple()
12+
"darwin-" * ARCH
13+
end
14+
15+
OUTPUT = "$(PLATFORM)-v$(VERSION.major)_$(VERSION.minor)_$(VERSION.patch).jl"
16+
OUTPUT_DEST = joinpath("src/memlayout", OUTPUT)
17+
18+
isfile(OUTPUT_DEST) && exit(0)
19+
20+
const PROJECT_DIR = (@__DIR__) |> dirname |> dirname |> dirname
21+
const INCLUDE = joinpath(dirname(Sys.BINDIR), "include/julia")
22+
const OPTIONS = if string(Sys.ARCH)[1] == 'i'
23+
Sys.islinux() && run(`sudo apt-get install g++-multilib -y`)
24+
["-std=c++11","-march=pentium4", "-m32", "-static-libgcc", "-static-libstdc++"]
25+
else
26+
["-std=c++11"]
27+
end
28+
29+
run(`g++ $(OPTIONS) -I$(INCLUDE) $(PROJECT_DIR)/.github/workflows/tasklayout/memlayout.cpp -o memlayout-gen.exe`)
30+
run(`./memlayout-gen.exe`)
31+
32+
isfile(OUTPUT) && Base.Filesystem.mv(OUTPUT, OUTPUT_DEST)

‎Project.toml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,18 @@ repo = "https://github.com/TuringLang/Libtask.jl.git"
66
version = "0.5.3"
77

88
[deps]
9+
Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
910
Libtask_jll = "3ae2931a-708c-5973-9c38-ccf7496fb450"
1011
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
1112
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
1213

1314
[compat]
14-
Libtask_jll = "0.4"
15+
Libtask_jll = "0.5.1"
1516
julia = "1.3"
1617

1718
[extras]
18-
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
1919
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
20+
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
2021

2122
[targets]
2223
test = ["Test", "BenchmarkTools"]

‎src/Libtask.jl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
11
module Libtask
22

3+
using Libdl
34
using Libtask_jll
45

56
export CTask, consume, produce, TArray, tzeros, tfill, TRef
67

8+
function __init__()
9+
@static if VERSION < v"1.6.0"
10+
push!(Libdl.DL_LOAD_PATH,
11+
Libtask_jll.get_libtask_julia_path() |> dirname)
12+
Libdl.dlopen(Libtask_jll.libtask_julia_path)
13+
end
14+
end
15+
16+
include("memlayout/main.jl")
717
include("ctask.jl")
818
include("tarray.jl")
919
include("tref.jl")

‎src/ctask.jl‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,18 @@ function enable_stack_copying(t::Task)
4949
if istaskfailed(t)
5050
error("only runnable or finished tasks' stack can be copied.")
5151
end
52-
return ccall((:jl_enable_stack_copying, libtask_julia), Any, (Any,), t)::Task
52+
# return ccall((:jl_enable_stack_copying, libtask_julia),
53+
# Any, (Any,), t)::Task
54+
copy_stack = internal_getfield(t, :copy_stack, Int32)
55+
if copy_stack == 0
56+
internal_setfield(t, :copy_stack, Int32(1))
57+
internal_setfield(t, :bufsz, Csize_t(0))
58+
ccall((:jl_reset_task_ctx, libtask_julia),
59+
Cvoid, (Any, Csize_t, Csize_t, Csize_t),
60+
t, TASK_OFFSETS[:ctx], TASK_OFFSETS[:sizeof_ctx],
61+
TASK_OFFSETS[:tls_base_context])
62+
end
63+
return t
5364
end
5465

5566
"""
@@ -101,7 +112,30 @@ function Base.copy(ctask::CTask)
101112
error("only runnable or finished tasks can be copied.")
102113
end
103114

104-
newtask = ccall((:jl_clone_task, libtask_julia), Any, (Any,), task)::Task
115+
# memory copy
116+
# newtask = ccall((:jl_clone_task, libtask_julia), Any, (Any,), task)::Task
117+
newtask = ccall((:jl_clone_task_opaque, libtask_julia),
118+
Any, (Any, Csize_t), task, TASK_OFFSETS[:END])::Task
119+
internal_setfield(newtask, :exception, nothing)
120+
internal_setfield(newtask, :backtrace, nothing)
121+
internal_setfield(newtask, :tls, nothing)
122+
internal_setfield(newtask, :result, nothing)
123+
internal_setfield(newtask, :donenotify, nothing)
124+
internal_setfield(newtask, :excstack, C_NULL)
125+
internal_setfield(newtask, :ptls, C_NULL)
126+
internal_setfield(newtask, :gcstack, C_NULL)
127+
128+
if haskey(TASK_OFFSETS, :stkbuf) && haskey(TASK_OFFSETS, :bufsz)
129+
old_stkbuf = internal_getfield(task, :stkbuf, Ptr)
130+
if old_stkbuf != C_NULL
131+
internal_setfield(task, :bufsz, Csize_t(0))
132+
else
133+
internal_setfield(newtask, :stkbuf, C_NULL)
134+
end
135+
internal_setfield(newtask, :bufsz, Csize_t(0))
136+
end
137+
memset(newtask, 0, :locks)
138+
# memory copy done
105139

106140
task.storage[:n_copies] = 1 + n_copies(task)
107141
newtask.storage = copy(task.storage)

‎src/memlayout/darwin-x86_64-v1_3_1.jl‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
ALL_TASK_OFFSETS[("darwin-x86_64", v"1.3.1")] = Dict(
2+
:END => 568,
3+
:backtrace => 56,
4+
:bufsz => 248,
5+
:copy_stack => 256,
6+
:ctx => 84,
7+
:donenotify => 32,
8+
:eh => 264,
9+
:exception => 48,
10+
:excstack => 280,
11+
:gcstack => 272,
12+
:locks => 304,
13+
:logstate => 64,
14+
:next => 0,
15+
:prio => 298,
16+
:queue => 8,
17+
:result => 40,
18+
:sizeof_ctx => 152,
19+
:start => 72,
20+
:state => 24,
21+
:sticky => 80,
22+
:stkbuf => 240,
23+
:tid => 296,
24+
:timing_stack => 560,
25+
:tls => 16,
26+
:tls_base_context => 6664,
27+
:world_age => 288,
28+
)

‎src/memlayout/darwin-x86_64-v1_4_2.jl‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
ALL_TASK_OFFSETS[("darwin-x86_64", v"1.4.2")] = Dict(
2+
:END => 568,
3+
:backtrace => 56,
4+
:bufsz => 248,
5+
:copy_stack => 256,
6+
:ctx => 84,
7+
:donenotify => 32,
8+
:eh => 264,
9+
:exception => 48,
10+
:excstack => 280,
11+
:gcstack => 272,
12+
:locks => 304,
13+
:logstate => 64,
14+
:next => 0,
15+
:prio => 298,
16+
:queue => 8,
17+
:result => 40,
18+
:sizeof_ctx => 152,
19+
:start => 72,
20+
:state => 24,
21+
:sticky => 80,
22+
:stkbuf => 240,
23+
:tid => 296,
24+
:timing_stack => 560,
25+
:tls => 16,
26+
:tls_base_context => 6664,
27+
:world_age => 288,
28+
)

0 commit comments

Comments
 (0)