[ Apr 14 13:42:05 Enabled. ]
[ Apr 14 13:42:05 Rereading configuration. ]
[ Apr 14 13:42:07 Rereading configuration. ]
[ Apr 14 13:42:13 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e7f0c25d-e783-42ad-a696-ff4846b3bdb0/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:42:13.504276345Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560}
{"msg":"dataset: \"oxp_e7f0c25d-e783-42ad-a696-ff4846b3bdb0/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:42:13.50496807Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:42:13.50497887Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:42:13.50498571Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:42:13.517724203Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:42:13.674831364Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.285474511Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:38209"}
{"msg":"region ab274be5-b7ce-45b7-9819-c43c213dcefc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.313633925Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.316123828Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.316138758Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions","method":"POST","req_id":"ea0a4523-e03e-4fac-9914-2c7c60f9c493","remote_addr":"[fd00:1122:3344:101::c]:38209","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":15369,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_e7f0c25d-e783-42ad-a696-ff4846b3bdb0/crucible/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.324930947Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_e7f0c25d-e783-42ad-a696-ff4846b3bdb0/crucible/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.324955118Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ab274be5-b7ce-45b7-9819-c43c213dcefc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.391114407Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"region":"ab274be5-b7ce-45b7-9819-c43c213dcefc","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ab274be5-b7ce-45b7-9819-c43c213dcefc\" \"--data\" \"/data/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.393462467Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"region":"ab274be5-b7ce-45b7-9819-c43c213dcefc","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.686648318Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions","method":"POST","req_id":"03ff9537-79d3-4cde-84ae-9836e7263377","remote_addr":"[fd00:1122:3344:101::c]:38209","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.721548687Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"region":"ab274be5-b7ce-45b7-9819-c43c213dcefc","component":"worker"}
{"msg":"region ab274be5-b7ce-45b7-9819-c43c213dcefc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.721573467Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.721685979Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ab274be5-b7ce-45b7-9819-c43c213dcefc","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.722020357Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ab274be5-b7ce-45b7-9819-c43c213dcefc","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.728421234Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.744326355Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ab274be5-b7ce-45b7-9819-c43c213dcefc","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.746913521Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.747483323Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.747546625Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.747588495Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.747626946Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.751240504Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:23.754928803Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:24.295274106Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51230"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:24.295422699Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","method":"GET","req_id":"705d185d-01f4-4dab-8977-944566b64df1","remote_addr":"[fd00:1122:3344:101::a]:51230","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:24.297760089Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:63729"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:24.297866941Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","method":"GET","req_id":"6aec83a1-894e-460e-854e-f4122e127a3e","remote_addr":"[fd00:1122:3344:101::b]:63729","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":35,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:24.345681517Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:64141"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:24.34582655Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","method":"GET","req_id":"f8c1388d-1c07-49c9-b001-dbd4b4f4c575","remote_addr":"[fd00:1122:3344:101::c]:64141","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":73,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:24.428391072Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions","method":"POST","req_id":"cb8d093b-e57d-4b11-907e-d3c88c973aea","remote_addr":"[fd00:1122:3344:101::c]:38209","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.217823589Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","method":"GET","req_id":"c1bd2ac1-9dd4-447e-a337-32ead60527ea","remote_addr":"[fd00:1122:3344:101::c]:38209","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":49,"response_code":200}
{"msg":"path is \"/data/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.221066748Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"datafile"}
{"msg":"dataset is oxp_e7f0c25d-e783-42ad-a696-ff4846b3bdb0/crucible/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.229459289Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"datafile"}
{"msg":"region ab274be5-b7ce-45b7-9819-c43c213dcefc state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.23699287Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.237113543Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.237135613Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","method":"DELETE","req_id":"960de94b-ac78-4c0c-ba82-54e625c27e7d","remote_addr":"[fd00:1122:3344:101::c]:38209","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":16066,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-ab274be5-b7ce-45b7-9819-c43c213dcefc (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.238286688Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.245037113Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","method":"GET","req_id":"2f275f82-ab7b-4223-b6b3-cd7793d09efd","remote_addr":"[fd00:1122:3344:101::c]:38209","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":34,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.245646876Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e7f0c25d-e783-42ad-a696-ff4846b3bdb0/crucible/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.254372363Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"region":"ab274be5-b7ce-45b7-9819-c43c213dcefc","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.309139788Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","method":"GET","req_id":"a5988a5c-8c3b-42ab-a576-54ff0443cf82","remote_addr":"[fd00:1122:3344:101::c]:38209","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"region ab274be5-b7ce-45b7-9819-c43c213dcefc state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.334307758Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T13:43:25.38842633Z","hostname":"oxz_crucible_ddbc9670-8b55-4d96-91e6-28f4e6bdfcda","pid":24560,"uri":"/crucible/0/regions/ab274be5-b7ce-45b7-9819-c43c213dcefc","method":"GET","req_id":"777d0fec-b9b9-483d-b90e-953cd016ed6c","remote_addr":"[fd00:1122:3344:101::c]:38209","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":50,"response_code":200}