[ Mar 31 01:55:43 Enabled. ]
[ Mar 31 01:55:43 Rereading configuration. ]
[ Mar 31 01:55:47 Rereading configuration. ]
[ Mar 31 01:55:57 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_74e08e61-7f99-4916-9d14-b3e50fd2bc4a/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 3df8a24e046bcfdb0d69f93075e8e1d2d0e9889d\nCommit timestamp: 2026-03-30T16:20:36.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:55:58.270690125Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836}
{"msg":"dataset: \"oxp_74e08e61-7f99-4916-9d14-b3e50fd2bc4a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:55:58.273837111Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:55:58.273888132Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:55:58.273894943Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:55:58.284179976Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:55:58.381126176Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.48956909Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37578"}
{"msg":"region 7ab5b922-39ba-432a-9868-c0f265951818 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.534586933Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.535363217Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.535411228Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"uri":"/crucible/0/regions","method":"POST","req_id":"954de67b-75fb-4c0a-b10f-e7dba6cc1ab5","remote_addr":"[fd00:1122:3344:101::c]:37578","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22162,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_74e08e61-7f99-4916-9d14-b3e50fd2bc4a/crucible/regions/7ab5b922-39ba-432a-9868-c0f265951818","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.541911074Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_74e08e61-7f99-4916-9d14-b3e50fd2bc4a/crucible/regions/7ab5b922-39ba-432a-9868-c0f265951818","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.541929164Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7ab5b922-39ba-432a-9868-c0f265951818\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7ab5b922-39ba-432a-9868-c0f265951818\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.615201892Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"region":"7ab5b922-39ba-432a-9868-c0f265951818","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7ab5b922-39ba-432a-9868-c0f265951818\" \"--data\" \"/data/regions/7ab5b922-39ba-432a-9868-c0f265951818\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.617431451Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"region":"7ab5b922-39ba-432a-9868-c0f265951818","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.747055384Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"uri":"/crucible/0/regions","method":"POST","req_id":"887e50ed-3a48-433e-a42c-ca4e8271ab0c","remote_addr":"[fd00:1122:3344:101::c]:37578","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.918634256Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"region":"7ab5b922-39ba-432a-9868-c0f265951818","component":"worker"}
{"msg":"region 7ab5b922-39ba-432a-9868-c0f265951818 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.918712547Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.918928791Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7ab5b922-39ba-432a-9868-c0f265951818","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.919278407Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7ab5b922-39ba-432a-9868-c0f265951818","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.928103575Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.943319886Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7ab5b922-39ba-432a-9868-c0f265951818","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.947727405Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7ab5b922-39ba-432a-9868-c0f265951818","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.948394827Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.948540579Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.94860125Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.948638451Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.952899557Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:56:59.957570061Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.465677637Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"uri":"/crucible/0/regions","method":"POST","req_id":"f4f88831-b9d6-4e1e-af62-1713a1c62212","remote_addr":"[fd00:1122:3344:101::c]:37578","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.854553786Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"uri":"/crucible/0/regions/7ab5b922-39ba-432a-9868-c0f265951818","method":"GET","req_id":"324b04d3-8909-4834-aa23-54155d9a9834","remote_addr":"[fd00:1122:3344:101::c]:37578","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":200}
{"msg":"path is \"/data/regions/7ab5b922-39ba-432a-9868-c0f265951818\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.857528879Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"datafile"}
{"msg":"dataset is oxp_74e08e61-7f99-4916-9d14-b3e50fd2bc4a/crucible/regions/7ab5b922-39ba-432a-9868-c0f265951818","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.865350908Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"datafile"}
{"msg":"region 7ab5b922-39ba-432a-9868-c0f265951818 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.872369663Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.872550187Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.872593717Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"uri":"/crucible/0/regions/7ab5b922-39ba-432a-9868-c0f265951818","method":"DELETE","req_id":"543e6038-1bb4-4a3e-8e06-bc152efecf32","remote_addr":"[fd00:1122:3344:101::c]:37578","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":15044,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-7ab5b922-39ba-432a-9868-c0f265951818 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.873551965Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.877355282Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.881048828Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"uri":"/crucible/0/regions/7ab5b922-39ba-432a-9868-c0f265951818","method":"GET","req_id":"f97b2d17-560d-4aff-be88-1acd716f738a","remote_addr":"[fd00:1122:3344:101::c]:37578","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":30,"response_code":200}
{"msg":"deleting zfs dataset \"oxp_74e08e61-7f99-4916-9d14-b3e50fd2bc4a/crucible/regions/7ab5b922-39ba-432a-9868-c0f265951818\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.88508602Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"region":"7ab5b922-39ba-432a-9868-c0f265951818","component":"worker"}
{"msg":"region 7ab5b922-39ba-432a-9868-c0f265951818 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.961368212Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-31T01:57:00.999812337Z","hostname":"oxz_crucible_7c66510a-c326-43b5-85f7-f048d3bafa9a","pid":24836,"uri":"/crucible/0/regions/7ab5b922-39ba-432a-9868-c0f265951818","method":"GET","req_id":"5d809aac-dec8-4559-9e07-2a7ad5d2a4c6","remote_addr":"[fd00:1122:3344:101::c]:37578","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":49,"response_code":200}