[ Oct 10 05:39:28 Enabled. ]
[ Oct 10 05:39:28 Rereading configuration. ]
[ Oct 10 05:39:30 Rereading configuration. ]
[ Oct 10 05:39:33 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_dc34e056-a9a5-45e9-ac9e-76d0c390b4d1/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_dc34e056-a9a5-45e9-ac9e-76d0c390b4d1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:39:33.829004852Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:39:33.830918093Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:39:33.830933713Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:39:33.847342235Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:39:34.067125418Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.657006448Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:44459"}
{"msg":"region f6427c2c-e89d-4b8b-b824-d7248cfa8d1d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.690767627Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.697261187Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"uri":"/crucible/0/regions","method":"POST","req_id":"846165b4-fc70-4a7f-8bf7-957e1ca78b06","remote_addr":"[fd00:1122:3344:101::c]:44459","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":15739,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.697281888Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_dc34e056-a9a5-45e9-ac9e-76d0c390b4d1/crucible/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.70646916Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_dc34e056-a9a5-45e9-ac9e-76d0c390b4d1/crucible/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.706525661Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f6427c2c-e89d-4b8b-b824-d7248cfa8d1d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.773625077Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"region":"f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f6427c2c-e89d-4b8b-b824-d7248cfa8d1d\" \"--data\" \"/data/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.780500558Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"region":"f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:01.943086656Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"uri":"/crucible/0/regions","method":"POST","req_id":"173e7c45-b78c-4b11-bb8b-242166caf78b","remote_addr":"[fd00:1122:3344:101::c]:44459","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.059404367Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"region":"f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","component":"worker"}
{"msg":"region f6427c2c-e89d-4b8b-b824-d7248cfa8d1d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.059468049Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.059662594Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.061957614Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.067182692Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.088639436Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.101057163Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.101675859Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.101762551Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.101806492Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.101862424Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.107819201Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.114374593Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.224609814Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"uri":"/crucible/0/regions","method":"POST","req_id":"165d3548-c801-4735-95a6-f4a7a38fb419","remote_addr":"[fd00:1122:3344:101::c]:44459","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.860727092Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"uri":"/crucible/0/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","method":"GET","req_id":"ff60c5de-2b7f-4107-a214-783436183822","remote_addr":"[fd00:1122:3344:101::c]:44459","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"path is \"/data/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.864806429Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"datafile"}
{"msg":"dataset is oxp_dc34e056-a9a5-45e9-ac9e-76d0c390b4d1/crucible/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.876755894Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"datafile"}
{"msg":"region f6427c2c-e89d-4b8b-b824-d7248cfa8d1d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.887772764Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.887895157Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.887972419Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"uri":"/crucible/0/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","method":"DELETE","req_id":"83b11cc4-b747-4819-95e7-fd9166a8d29e","remote_addr":"[fd00:1122:3344:101::c]:44459","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23201,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f6427c2c-e89d-4b8b-b824-d7248cfa8d1d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.889009466Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.89144204Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"uri":"/crucible/0/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","method":"GET","req_id":"3cd7fcd5-f49b-4bfc-a224-6ec9475eee4f","remote_addr":"[fd00:1122:3344:101::c]:44459","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.894251094Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_dc34e056-a9a5-45e9-ac9e-76d0c390b4d1/crucible/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.905379007Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"region":"f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","component":"worker"}
{"msg":"region f6427c2c-e89d-4b8b-b824-d7248cfa8d1d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:02.967614945Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T05:40:03.021509463Z","hostname":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","pid":23100,"uri":"/crucible/0/regions/f6427c2c-e89d-4b8b-b824-d7248cfa8d1d","method":"GET","req_id":"3639a5c0-a76b-4907-8337-f3e87ed5f2c0","remote_addr":"[fd00:1122:3344:101::c]:44459","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":86,"response_code":"200"}