[ Mar 17 16:40:55 Enabled. ]
[ Mar 17 16:40:55 Rereading configuration. ]
[ Mar 17 16:40:57 Rereading configuration. ]
[ Mar 17 16:41:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2343e368-175d-407b-9346-6639bf6b43b6/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_2343e368-175d-407b-9346-6639bf6b43b6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.67946985Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.68274075Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.68274869Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.699397167Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.771150764Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.025256013Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:53952"}
{"msg":"region 3b292ce4-3cea-4c18-8a78-705b089034ed state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.069610083Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.07178069Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"uri":"/crucible/0/regions","method":"POST","req_id":"2976c550-7861-470a-af3b-19e8e549e2f9","remote_addr":"[fd00:1122:3344:101::c]:53952","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19055,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.072399313Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_2343e368-175d-407b-9346-6639bf6b43b6/crucible/regions/3b292ce4-3cea-4c18-8a78-705b089034ed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.081720423Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_2343e368-175d-407b-9346-6639bf6b43b6/crucible/regions/3b292ce4-3cea-4c18-8a78-705b089034ed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.081753694Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3b292ce4-3cea-4c18-8a78-705b089034ed\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3b292ce4-3cea-4c18-8a78-705b089034ed\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.149495604Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"region":"3b292ce4-3cea-4c18-8a78-705b089034ed","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3b292ce4-3cea-4c18-8a78-705b089034ed\" \"--data\" \"/data/regions/3b292ce4-3cea-4c18-8a78-705b089034ed\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.151505207Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"region":"3b292ce4-3cea-4c18-8a78-705b089034ed","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.272234473Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"uri":"/crucible/0/regions","method":"POST","req_id":"f7310f71-e9c3-49c5-892f-04cbad10f10a","remote_addr":"[fd00:1122:3344:101::c]:53952","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.360904302Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"region":"3b292ce4-3cea-4c18-8a78-705b089034ed","component":"worker"}
{"msg":"region 3b292ce4-3cea-4c18-8a78-705b089034ed state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.360960303Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.361059875Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3b292ce4-3cea-4c18-8a78-705b089034ed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.361466474Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3b292ce4-3cea-4c18-8a78-705b089034ed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.368141317Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.382484834Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3b292ce4-3cea-4c18-8a78-705b089034ed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.386254315Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3b292ce4-3cea-4c18-8a78-705b089034ed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.38696011Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.387028021Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.387070673Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.387107923Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.391195811Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.404863934Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.84226399Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"uri":"/crucible/0/regions","method":"POST","req_id":"14de5a46-aa57-4c0a-92b0-87a61d6c2cd7","remote_addr":"[fd00:1122:3344:101::c]:53952","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":48,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.189816074Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"uri":"/crucible/0/regions/3b292ce4-3cea-4c18-8a78-705b089034ed","method":"GET","req_id":"d19e5295-3161-42ff-8e7c-381a52057da0","remote_addr":"[fd00:1122:3344:101::c]:53952","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":2093,"response_code":"200"}
{"msg":"path is \"/data/regions/3b292ce4-3cea-4c18-8a78-705b089034ed\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.192670195Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"datafile"}
{"msg":"dataset is oxp_2343e368-175d-407b-9346-6639bf6b43b6/crucible/regions/3b292ce4-3cea-4c18-8a78-705b089034ed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.205616212Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"datafile"}
{"msg":"region 3b292ce4-3cea-4c18-8a78-705b089034ed state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.216937965Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.217113189Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.21716707Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"uri":"/crucible/0/regions/3b292ce4-3cea-4c18-8a78-705b089034ed","method":"DELETE","req_id":"d8a2bbcf-42fe-4b40-badc-03ef9efc3b27","remote_addr":"[fd00:1122:3344:101::c]:53952","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24532,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-3b292ce4-3cea-4c18-8a78-705b089034ed (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.218321515Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.220541572Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"uri":"/crucible/0/regions/3b292ce4-3cea-4c18-8a78-705b089034ed","method":"GET","req_id":"80918639-9b63-4594-bc3e-266a27ae331e","remote_addr":"[fd00:1122:3344:101::c]:53952","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.223135358Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_2343e368-175d-407b-9346-6639bf6b43b6/crucible/regions/3b292ce4-3cea-4c18-8a78-705b089034ed\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.234755627Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"region":"3b292ce4-3cea-4c18-8a78-705b089034ed","component":"worker"}
{"msg":"region 3b292ce4-3cea-4c18-8a78-705b089034ed state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.308120448Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.308333732Z","hostname":"oxz_crucible_abe0c92b-5006-40fd-9f4e-fd05d72dd37d","pid":22363,"uri":"/crucible/0/regions/3b292ce4-3cea-4c18-8a78-705b089034ed","method":"GET","req_id":"63c19312-9b91-4173-a86d-74ab2fd235bf","remote_addr":"[fd00:1122:3344:101::c]:53952","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":101,"response_code":"200"}