[ Mar 11 22:06:51 Enabled. ]
[ Mar 11 22:06:51 Rereading configuration. ]
[ Mar 11 22:06:53 Rereading configuration. ]
[ Mar 11 22:06:59 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5b5107f2-6489-4da3-9ec4-67fd08c85b55/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_5b5107f2-6489-4da3-9ec4-67fd08c85b55/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:06:59.456534745Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:06:59.459282769Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:06:59.459291829Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:06:59.473283328Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:06:59.848157994Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:55.962731856Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63877"}
{"msg":"region a6605b16-558e-4dee-9df0-79c2e3230d7c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.03597106Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.047236176Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"uri":"/crucible/0/regions","method":"POST","req_id":"75a0ebf3-23cd-4f6e-8f7c-e4f8f920860e","remote_addr":"[fd00:1122:3344:101::a]:63877","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":39919,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.047270367Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_5b5107f2-6489-4da3-9ec4-67fd08c85b55/crucible/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.056878629Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_5b5107f2-6489-4da3-9ec4-67fd08c85b55/crucible/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.056915369Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a6605b16-558e-4dee-9df0-79c2e3230d7c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.170739829Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"region":"a6605b16-558e-4dee-9df0-79c2e3230d7c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a6605b16-558e-4dee-9df0-79c2e3230d7c\" \"--data\" \"/data/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.173835516Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"region":"a6605b16-558e-4dee-9df0-79c2e3230d7c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.225461716Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"uri":"/crucible/0/regions","method":"POST","req_id":"13fff55e-678f-4ed8-b05d-a8f1a7cd53dd","remote_addr":"[fd00:1122:3344:101::a]:63877","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.522399367Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"region":"a6605b16-558e-4dee-9df0-79c2e3230d7c","component":"worker"}
{"msg":"region a6605b16-558e-4dee-9df0-79c2e3230d7c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.522447187Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.522570748Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a6605b16-558e-4dee-9df0-79c2e3230d7c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.523059833Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a6605b16-558e-4dee-9df0-79c2e3230d7c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.544132362Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.574827024Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a6605b16-558e-4dee-9df0-79c2e3230d7c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.581613162Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.582358228Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.582424309Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.582468879Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.58250733Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.594994326Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.625642277Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:56.708905877Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"uri":"/crucible/0/regions","method":"POST","req_id":"260f2695-4985-4c50-a30c-490fa3a5ce07","remote_addr":"[fd00:1122:3344:101::a]:63877","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.318555833Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"uri":"/crucible/0/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","method":"GET","req_id":"19ba2ae6-9463-46ec-aa38-a537dc170b81","remote_addr":"[fd00:1122:3344:101::a]:63877","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1976,"response_code":"200"}
{"msg":"path is \"/data/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.320981814Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"datafile"}
{"msg":"dataset is oxp_5b5107f2-6489-4da3-9ec4-67fd08c85b55/crucible/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.331912117Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"datafile"}
{"msg":"region a6605b16-558e-4dee-9df0-79c2e3230d7c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.341447088Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.34161478Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.34164884Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"uri":"/crucible/0/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","method":"DELETE","req_id":"6c75e4b2-8511-4213-9443-22343ec03b35","remote_addr":"[fd00:1122:3344:101::a]:63877","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20704,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-a6605b16-558e-4dee-9df0-79c2e3230d7c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.342663159Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.344667506Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"uri":"/crucible/0/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","method":"GET","req_id":"f1ccebf2-5dca-42bc-a42f-080633848697","remote_addr":"[fd00:1122:3344:101::a]:63877","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.358394033Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5b5107f2-6489-4da3-9ec4-67fd08c85b55/crucible/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.369181425Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"region":"a6605b16-558e-4dee-9df0-79c2e3230d7c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.448770863Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"uri":"/crucible/0/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","method":"GET","req_id":"1fff5b0a-07a9-4ea6-900d-5f234b906937","remote_addr":"[fd00:1122:3344:101::a]:63877","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region a6605b16-558e-4dee-9df0-79c2e3230d7c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.478215004Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-11T22:07:57.539004292Z","hostname":"oxz_crucible_47cdcf0c-13b5-4216-ba10-0a2e76e85588","pid":22206,"uri":"/crucible/0/regions/a6605b16-558e-4dee-9df0-79c2e3230d7c","method":"GET","req_id":"1bd1a73d-7de7-4b66-9575-14f0b5e71b42","remote_addr":"[fd00:1122:3344:101::a]:63877","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28,"response_code":"200"}