[ Sep 23 04:31:11 Enabled. ] [ Sep 23 04:31:11 Rereading configuration. ] [ Sep 23 04:31:14 Rereading configuration. ] [ Sep 23 04:31:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 779775d5130ff7a4836f52f48b7e64d1479ee104\nCommit timestamp: 2025-08-09T00:27:59.000000000Z branch: main\nrustc: 1.86.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:31:20.333336449Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020} {"msg":"dataset: \"oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:31:20.336506114Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020} {"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:31:20.336530084Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:31:20.336537555Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:31:20.354792737Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:31:20.438402597Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.590212931Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42313"} {"msg":"region 4011023c-c1b5-4211-82ea-670c668ef0f3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.621694437Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.624057293Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions","method":"POST","req_id":"4cca8d09-5d96-4bd3-9be4-a26ebc49dd6d","remote_addr":"[fd00:1122:3344:101::c]:42313","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":13607,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.626315706Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.635395562Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.635425632Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"4011023c-c1b5-4211-82ea-670c668ef0f3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.690541258Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"region":"4011023c-c1b5-4211-82ea-670c668ef0f3","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4011023c-c1b5-4211-82ea-670c668ef0f3\" \"--data\" \"/data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.692810362Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"region":"4011023c-c1b5-4211-82ea-670c668ef0f3","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:16.903982604Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions","method":"POST","req_id":"0bcba0dc-8dc9-4bce-a1cd-081f3f348367","remote_addr":"[fd00:1122:3344:101::c]:42313","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":68,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.015846853Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"region":"4011023c-c1b5-4211-82ea-670c668ef0f3","component":"worker"} {"msg":"region 4011023c-c1b5-4211-82ea-670c668ef0f3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.015891275Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.015998577Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.016368266Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.023833343Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.040527338Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.043948239Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.044486462Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.052494102Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.052544893Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.052583244Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.057832128Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.062425627Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:32:17.501574828Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions","method":"POST","req_id":"089806d2-a964-476e-a65a-a2fc4ba3fd8c","remote_addr":"[fd00:1122:3344:101::c]:42313","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":48,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:58.880115899Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45620"} {"msg":"region 319d94b8-1843-4a90-8316-52dc81007cba state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:58.880709373Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:58.881039561Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions","method":"POST","req_id":"e0a09e9c-3255-4142-afbf-2bae1bf58404","remote_addr":"[fd00:1122:3344:101::c]:45620","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":630,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:58.881060232Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/319d94b8-1843-4a90-8316-52dc81007cba","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:58.891194442Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/319d94b8-1843-4a90-8316-52dc81007cba","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:58.891233153Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"319d94b8-1843-4a90-8316-52dc81007cba\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/319d94b8-1843-4a90-8316-52dc81007cba\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:58.963366181Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"region":"319d94b8-1843-4a90-8316-52dc81007cba","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"319d94b8-1843-4a90-8316-52dc81007cba\" \"--data\" \"/data/regions/319d94b8-1843-4a90-8316-52dc81007cba\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:58.965258636Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"region":"319d94b8-1843-4a90-8316-52dc81007cba","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.092564802Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"region":"319d94b8-1843-4a90-8316-52dc81007cba","component":"worker"} {"msg":"region 319d94b8-1843-4a90-8316-52dc81007cba state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.092606433Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.092703145Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-319d94b8-1843-4a90-8316-52dc81007cba","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.093082054Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-319d94b8-1843-4a90-8316-52dc81007cba","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.103359457Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.127430808Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-319d94b8-1843-4a90-8316-52dc81007cba","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.130562822Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/319d94b8-1843-4a90-8316-52dc81007cba","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.130875409Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.130928151Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.130965451Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.130999342Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.134987007Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145302171Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145386043Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145469385Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145479945Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145518786Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145563907Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145650159Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.14570235Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145747291Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.145790343Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.155681047Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.156116517Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:42:59.244530331Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions","method":"POST","req_id":"ba0e2877-deac-469a-952e-e3ae94ea4864","remote_addr":"[fd00:1122:3344:101::c]:45620","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":68,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.678879067Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","method":"GET","req_id":"794dbe45-c984-4809-a94b-8d70e97c0546","remote_addr":"[fd00:1122:3344:101::c]:45620","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":50,"response_code":200} {"msg":"path is \"/data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.684294965Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"dataset is oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.693812641Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"snapshot is oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3@12824763-bda7-46fe-bcd9-8766e883ee0e","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.703658274Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3\", \"12824763-bda7-46fe-bcd9-8766e883ee0e\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.703693625Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.71702805Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions/4011023c-c1b5-4211-82ea-670c668ef0f3/snapshots/12824763-bda7-46fe-bcd9-8766e883ee0e","method":"GET","req_id":"9e5029ef-592f-4631-87a3-e3bc3bd12a6f","remote_addr":"[fd00:1122:3344:101::c]:45620","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":34980,"response_code":200} {"msg":"path is \"/data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.720280998Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"dataset is oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.728589534Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"snapshot is oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3@12824763-bda7-46fe-bcd9-8766e883ee0e","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.736927472Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3\", \"12824763-bda7-46fe-bcd9-8766e883ee0e\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.736963093Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 4011023c-c1b5-4211-82ea-670c668ef0f3-12824763-bda7-46fe-bcd9-8766e883ee0e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.747364459Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"applying SMF actions for region 4011023c-c1b5-4211-82ea-670c668ef0f3 running snapshot 12824763-bda7-46fe-bcd9-8766e883ee0e (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.747556334Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.747581134Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions/4011023c-c1b5-4211-82ea-670c668ef0f3/snapshots/12824763-bda7-46fe-bcd9-8766e883ee0e/run","method":"POST","req_id":"13eba596-c356-44ed-bf3a-6424f20821cd","remote_addr":"[fd00:1122:3344:101::c]:45620","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":27344,"response_code":200} {"msg":"existing directory value /dev/null does not match /data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.748999808Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.74907449Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.749147501Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.749154231Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.749191202Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.749233253Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.74952876Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.749583192Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.749628443Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.749670584Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.753777371Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-4011023c-c1b5-4211-82ea-670c668ef0f3-12824763-bda7-46fe-bcd9-8766e883ee0e","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.754214901Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-4011023c-c1b5-4211-82ea-670c668ef0f3-12824763-bda7-46fe-bcd9-8766e883ee0e","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.762219731Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-4011023c-c1b5-4211-82ea-670c668ef0f3-12824763-bda7-46fe-bcd9-8766e883ee0e","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.772397972Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3/.zfs/snapshot/12824763-bda7-46fe-bcd9-8766e883ee0e","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.772660808Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.77271762Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.772756561Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.772792631Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.772842173Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.777028152Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.782413209Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"worker"} {"msg":"region 4011023c-c1b5-4211-82ea-670c668ef0f3 running snapshot 12824763-bda7-46fe-bcd9-8766e883ee0e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:00.78242886Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"path is \"/data/regions/4011023c-c1b5-4211-82ea-670c668ef0f3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:01.013049922Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"dataset is oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:01.022258431Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"datafile"} {"msg":"snapshot is oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3@12824763-bda7-46fe-bcd9-8766e883ee0e","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:01.032111484Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3e402de6-4ecb-4050-932d-065e3279c012/crucible/regions/4011023c-c1b5-4211-82ea-670c668ef0f3\", \"12824763-bda7-46fe-bcd9-8766e883ee0e\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:01.032154795Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-23T04:43:01.041377153Z","hostname":"oxz_crucible_1d5cb3fa-5620-4acf-8811-787a13247e59","pid":26020,"uri":"/crucible/0/regions/4011023c-c1b5-4211-82ea-670c668ef0f3/snapshots/12824763-bda7-46fe-bcd9-8766e883ee0e/run","method":"POST","req_id":"6b685150-177a-459d-9641-47a1b8917d12","remote_addr":"[fd00:1122:3344:101::c]:45620","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":28353,"response_code":200}