[ Feb 28 07:44:01 Enabled. ]
[ Feb 28 07:44:01 Rereading configuration. ]
[ Feb 28 07:44:04 Rereading configuration. ]
[ Feb 28 07:44:07 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:07.972439621Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:07.977255154Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:07.977272475Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:08.310195614Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.503393563Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:47288"}
{"msg":"region 167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.565078423Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.566052069Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"uri":"/crucible/0/regions","method":"POST","req_id":"6f79f945-ce15-4f04-9d81-a86e031d2543","remote_addr":"[fd00:1122:3344:101::b]:47288","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":3944,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.56610396Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.575500873Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.575561964Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.649775931Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"region":"167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.6987751Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"uri":"/crucible/0/regions","method":"POST","req_id":"7755e4e1-9344-407c-bcd8-0f68742eb1ed","remote_addr":"[fd00:1122:3344:101::b]:47288","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.772526638Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"region":"167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","component":"worker"}
{"msg":"region 167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.772563108Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.772779462Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.77326328Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.777193239Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.785835278Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.789121605Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.789873979Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.78995908Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.790004571Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.790040841Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.794931016Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.80093791Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.086594211Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"uri":"/crucible/0/regions","method":"POST","req_id":"9288a19a-2505-46f8-a23c-ca8e4c9ae9e1","remote_addr":"[fd00:1122:3344:101::b]:47288","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.86613418Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:48456"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.866288023Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"uri":"/crucible/0/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","method":"GET","req_id":"5760e3b2-2424-4b85-9bbb-e6bf576b0c11","remote_addr":"[fd00:1122:3344:101::b]:48456","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":112,"response_code":"200"}
{"msg":"path is \"/data/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.867072077Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.87704554Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"datafile"}
{"msg":"region 167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.885601348Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.885813281Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.885887443Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"uri":"/crucible/0/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","method":"DELETE","req_id":"ad9b0ac8-9747-484a-835d-9a41e7f98bc2","remote_addr":"[fd00:1122:3344:101::b]:48456","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19389,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.886045076Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"uri":"/crucible/0/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","method":"GET","req_id":"ccabd641-3042-4a74-a56c-0c60e8554cf0","remote_addr":"[fd00:1122:3344:101::b]:48456","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.886826259Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.891196635Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.900550147Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"region":"167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","component":"worker"}
{"msg":"region 167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.96191455Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:49.004549649Z","hostname":"oxz_crucible_f63ea677-b8c2-4e54-bd5d-b9613727817a","pid":22011,"uri":"/crucible/0/regions/167bd7b0-d3d2-4de2-8431-0aea6e4fe1b1","method":"GET","req_id":"c01e6ee9-da9e-4178-addf-045c7c1b6dc1","remote_addr":"[fd00:1122:3344:101::b]:48456","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":56,"response_code":"200"}