[ Dec 20 04:54:07 Enabled. ]
[ Dec 20 04:54:07 Rereading configuration. ]
[ Dec 20 04:54:09 Rereading configuration. ]
[ Dec 20 04:54:14 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3a5515a9-e40e-48af-88eb-4aade20355f4/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:14.953094754Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506}
{"msg":"dataset: \"oxp_3a5515a9-e40e-48af-88eb-4aade20355f4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:14.953301607Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:14.953315167Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:14.953320327Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:14.970661671Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:54:15.186996443Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.747806432Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60106"}
{"msg":"region 15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.791420716Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.794499613Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.794526623Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions","method":"POST","req_id":"43361ab5-9f0f-42f2-8363-18889bd07cf5","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19720,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_3a5515a9-e40e-48af-88eb-4aade20355f4/crucible/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.804270242Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_3a5515a9-e40e-48af-88eb-4aade20355f4/crucible/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.804297572Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.89482849Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"region":"15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c\" \"--data\" \"/data/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:09.897888476Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"region":"15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.055678477Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions","method":"POST","req_id":"067a616a-f40f-4ebe-ad73-080c31e6b179","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.352347312Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions","method":"POST","req_id":"f7a1ac3a-3b48-4381-8fc2-88b5794a1349","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.749576566Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"region":"15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","component":"worker"}
{"msg":"region 15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.749615947Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.749775589Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.750321908Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.828799662Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.967167078Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.993041431Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.993691081Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.993756442Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.993797853Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:10.993836453Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:11.019533284Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:11.046556506Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:11.839642244Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions","method":"POST","req_id":"4cc9cccb-f4f6-4813-a7ad-9776aededb02","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":97,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.152373189Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","method":"GET","req_id":"e7f9b169-6c09-4a5b-8445-1a075fcf161c","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":86,"response_code":200}
{"msg":"path is \"/data/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.155295944Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"datafile"}
{"msg":"dataset is oxp_3a5515a9-e40e-48af-88eb-4aade20355f4/crucible/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.163145303Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"datafile"}
{"msg":"region 15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.171007233Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.171137175Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","method":"DELETE","req_id":"22cbedbd-7e80-4ae5-beb4-59769163fd7e","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":15867,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.171170305Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.17216554Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.190011642Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","method":"GET","req_id":"fb2e4a6d-16e7-495a-b133-4bd1d92cdeb4","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.202300249Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3a5515a9-e40e-48af-88eb-4aade20355f4/crucible/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.210095008Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"region":"15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.33704813Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","method":"GET","req_id":"885053f1-3550-464d-8841-61dec18386a3","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.480815057Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","method":"GET","req_id":"0fe0b568-1150-4639-b800-2a5b92a13a93","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":42,"response_code":200}
{"msg":"region 15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.640317844Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-20T04:55:13.640656689Z","hostname":"oxz_crucible_371601a5-5a05-43be-8640-9a6fd36be0e0","pid":23506,"uri":"/crucible/0/regions/15a3b4f8-9647-4cce-9a4e-6c47bf8e4f1c","method":"GET","req_id":"cdeac8c9-82b5-4554-b6b2-14f27ffc7e3a","remote_addr":"[fd00:1122:3344:101::c]:60106","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":42,"response_code":200}