[ Sep 3 04:03:46 Enabled. ] [ Sep 3 04:03:46 Rereading configuration. ] [ Sep 3 04:03:51 Rereading configuration. ] [ Sep 3 04:03:59 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_829bea3e-8a89-4a4c-94e7-3aaa8e8b1a70/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 779775d5130ff7a4836f52f48b7e64d1479ee104\nCommit timestamp: 2025-08-09T00:27:59.000000000Z branch: main\nrustc: 1.86.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:03:59.983383436Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981} {"msg":"dataset: \"oxp_829bea3e-8a89-4a4c-94e7-3aaa8e8b1a70/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:03:59.986305617Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981} {"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:03:59.986350648Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:03:59.986357808Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:00.00333815Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:00.087066853Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.33710306Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54887"} {"msg":"region ad508801-ad85-4fe7-b297-dc7dff8595bd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.376509309Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.380899411Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"uri":"/crucible/0/regions","method":"POST","req_id":"b8d4ca6a-adfe-43de-bf55-8e0de59b0eaf","remote_addr":"[fd00:1122:3344:101::c]:54887","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20961,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.383832452Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_829bea3e-8a89-4a4c-94e7-3aaa8e8b1a70/crucible/regions/ad508801-ad85-4fe7-b297-dc7dff8595bd","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.393435792Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_829bea3e-8a89-4a4c-94e7-3aaa8e8b1a70/crucible/regions/ad508801-ad85-4fe7-b297-dc7dff8595bd","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.393463352Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"ad508801-ad85-4fe7-b297-dc7dff8595bd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ad508801-ad85-4fe7-b297-dc7dff8595bd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.454501405Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"region":"ad508801-ad85-4fe7-b297-dc7dff8595bd","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ad508801-ad85-4fe7-b297-dc7dff8595bd\" \"--data\" \"/data/regions/ad508801-ad85-4fe7-b297-dc7dff8595bd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.456828852Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"region":"ad508801-ad85-4fe7-b297-dc7dff8595bd","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.713432725Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"uri":"/crucible/0/regions","method":"POST","req_id":"88223d94-9479-4d13-b5ac-5300b9a14ef5","remote_addr":"[fd00:1122:3344:101::c]:54887","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.763565679Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"region":"ad508801-ad85-4fe7-b297-dc7dff8595bd","component":"worker"} {"msg":"region ad508801-ad85-4fe7-b297-dc7dff8595bd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.763622449Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.763851431Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-ad508801-ad85-4fe7-b297-dc7dff8595bd","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.764237524Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ad508801-ad85-4fe7-b297-dc7dff8595bd","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.769379451Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.785597809Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ad508801-ad85-4fe7-b297-dc7dff8595bd","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.788901253Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ad508801-ad85-4fe7-b297-dc7dff8595bd","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.789599578Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.797544686Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.797599946Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.797643716Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.804611637Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:36.808518746Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-03T04:04:37.20858854Z","hostname":"oxz_crucible_ff35182d-6daa-4bd3-80a4-b9b4d2f0224b","pid":23981,"uri":"/crucible/0/regions","method":"POST","req_id":"421d28e9-329f-4c6d-b6b0-c998f9b44ef6","remote_addr":"[fd00:1122:3344:101::c]:54887","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":45,"response_code":200}