A vibe coded tangled fork which supports pijul.
spindle/engine: read secrets from environment file
authored by
jobala
and committed by
tangled.org
6b0bc344
e952dd1d
+36
-25
+17
-4
nix/modules/spindle.nix
+17
-4
nix/modules/spindle.nix
···
116
116
description = "S3 bucket for workflow logs";
117
117
};
118
118
};
119
+
120
+
environmentFile = mkOption {
121
+
type = with types; nullOr path;
122
+
default = null;
123
+
example = "/etc/spindle.env";
124
+
description = ''
125
+
Additional environment file as defined in {manpage}`systemd.exec(5)`.
126
+
127
+
Sensitive secrets such as {env}`AWS_SECRET_ACCESS_KEY`,
128
+
{env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION`
129
+
may be passed to the service
130
+
without making them world readable in the nix store.
131
+
'';
132
+
};
119
133
};
120
134
};
121
135
···
129
143
serviceConfig = {
130
144
LogsDirectory = "spindle";
131
145
StateDirectory = "spindle";
146
+
EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
147
+
132
148
Environment = [
133
149
"SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}"
134
150
"SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}"
···
144
160
"SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"
145
161
"SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}"
146
162
"SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
147
-
"SPINDLE_NIXERY_PIPELINES_LOG_BUCKET=${cfg.pipelines.logBucket}"
148
-
"AWS_ACCESS_KEY_ID=${builtins.getEnv "AWS_ACCESS_KEY_ID"}"
149
-
"AWS_SECRET_ACCESS_KEY=${builtins.getEnv "AWS_SECRET_ACCESS_KEY"}"
150
-
"AWS_REGION=${builtins.getEnv "AWS_REGION"}"
163
+
"SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}"
151
164
];
152
165
ExecStart = "${cfg.package}/bin/spindle";
153
166
Restart = "always";
+5
nix/vm.nix
+5
nix/vm.nix
···
118
118
};
119
119
services.tangled.spindle = {
120
120
enable = true;
121
+
environmentFile = "/var/lib/spindle/.env";
121
122
server = {
122
123
owner = envVar "TANGLED_VM_SPINDLE_OWNER";
123
124
hostname = envVarOr "TANGLED_VM_SPINDLE_HOST" "localhost:6555";
···
130
131
secrets = {
131
132
provider = "sqlite";
132
133
};
134
+
};
135
+
136
+
pipelines = {
137
+
logBucket = envVarOr "SPINDLE_S3_LOG_BUCKET" "";
133
138
};
134
139
};
135
140
services.postgresql = {
+7
-18
spindle/engine/engine.go
+7
-18
spindle/engine/engine.go
···
37
37
secretValues[i] = s.Value
38
38
}
39
39
40
+
s3, err := NewS3(cfg.S3.LogBucket)
41
+
if err != nil {
42
+
l.Error("error creating s3 client", "err", err)
43
+
}
44
+
40
45
var wg sync.WaitGroup
41
46
for eng, wfs := range pipeline.Workflows {
42
47
workflowTimeout := eng.WorkflowTimeout()
···
53
58
}
54
59
55
60
defer func() {
56
-
logBucket := cfg.S3.LogBucket
57
-
58
-
if logBucket != "" {
61
+
if s3 != nil {
59
62
logFile := filepath.Join(cfg.Server.LogDir, fmt.Sprintf("%s.log", wid.String()))
60
-
if err := uploadWorkflowLogs(ctx, logFile, "tangled-demo"); err != nil {
63
+
if err := s3.WriteFile(ctx, logFile); err != nil {
61
64
l.Error("error uploading logs", "err", err)
62
65
}
63
66
}
···
144
147
wg.Wait()
145
148
l.Info("all workflows completed")
146
149
}
147
-
148
-
func uploadWorkflowLogs(ctx context.Context, logfile, bucket string) error {
149
-
s3, err := NewS3(bucket)
150
-
if err != nil {
151
-
return fmt.Errorf("error creating s3 client: %w", err)
152
-
}
153
-
154
-
name := filepath.Join(logfile)
155
-
if err := s3.WriteFile(ctx, name); err != nil {
156
-
return fmt.Errorf("error saving logs: %w", err)
157
-
}
158
-
159
-
return nil
160
-
}
+7
-3
spindle/engine/s3.go
+7
-3
spindle/engine/s3.go
···
17
17
client *s3.Client
18
18
}
19
19
20
-
const BASE_S3_PATH = "spindle/workflows"
20
+
const BaseS3Path = "spindle/workflows"
21
21
22
22
func NewS3(bucket string) (*S3, error) {
23
+
if bucket == "" {
24
+
return nil, fmt.Errorf("s3 bucket not provided")
25
+
}
26
+
23
27
ctx := context.Background()
24
28
sdkConfig, err := config.LoadDefaultConfig(ctx)
25
29
···
35
39
}
36
40
37
41
func (s *S3) WriteFile(ctx context.Context, path string) error {
38
-
s3_key := fmt.Sprintf("%s/%s", BASE_S3_PATH, filepath.Base(path))
42
+
s3Key := fmt.Sprintf("%s/%s", BaseS3Path, filepath.Base(path))
39
43
40
44
file, err := os.Open(path)
41
45
if err != nil {
···
45
49
46
50
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
47
51
Bucket: &s.bucket,
48
-
Key: &s3_key,
52
+
Key: &s3Key,
49
53
Body: file,
50
54
})
51
55