[ Jul 16 03:48:47 Enabled. ]
[ Jul 16 03:48:47 Rereading configuration. ]
[ Jul 16 03:48:49 Rereading configuration. ]
[ Jul 16 03:48:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:48:53.449374649Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:48:53.453017979Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:48:53.453030529Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:48:53.473242439Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:48:53.626649035Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.40939585Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33932"}
{"msg":"region 92c00360-cfdb-42cd-9ddd-6d1f24359322 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.430643725Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.439557464Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.439571374Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions","method":"POST","req_id":"4319ec39-cb00-4dd3-95e1-62e377cfe15d","remote_addr":"[fd00:1122:3344:101::a]:33932","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17952,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.452196763Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.452227993Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"92c00360-cfdb-42cd-9ddd-6d1f24359322\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.49952414Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"region":"92c00360-cfdb-42cd-9ddd-6d1f24359322","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"92c00360-cfdb-42cd-9ddd-6d1f24359322\" \"--data\" \"/data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.501448221Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"region":"92c00360-cfdb-42cd-9ddd-6d1f24359322","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.626763043Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions","method":"POST","req_id":"9c4c0f8f-e38e-4a17-8c28-a71476335542","remote_addr":"[fd00:1122:3344:101::a]:33932","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.658579466Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"region":"92c00360-cfdb-42cd-9ddd-6d1f24359322","component":"worker"}
{"msg":"region 92c00360-cfdb-42cd-9ddd-6d1f24359322 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.658602396Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.658748137Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.65921294Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.665354093Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.678791296Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.681832283Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.682537157Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.68681835Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.6868802Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.686930431Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.700521405Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:09.707213371Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:10.273175162Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:32870"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:10.273291833Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","method":"GET","req_id":"bb1d85a0-a591-4149-8206-46c4f0944c6f","remote_addr":"[fd00:1122:3344:101::b]:32870","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:50:10.365070302Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions","method":"POST","req_id":"f54f35b5-2af9-48b5-a15c-06d0ac3a711e","remote_addr":"[fd00:1122:3344:101::a]:33932","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":605,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.690486515Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42391"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.690604045Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","method":"GET","req_id":"38ffe67b-2c13-43ea-a73d-e907e142f839","remote_addr":"[fd00:1122:3344:101::a]:42391","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"path is \"/data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.693855943Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"dataset is oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.702948352Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"snapshot is oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322@8546765b-299a-4074-b332-a3f1ba0d9bb8","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.711078517Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322\", \"8546765b-299a-4074-b332-a3f1ba0d9bb8\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.711115997Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.722155697Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322/snapshots/8546765b-299a-4074-b332-a3f1ba0d9bb8","method":"GET","req_id":"45785f9b-0411-459b-8407-15477722a1bb","remote_addr":"[fd00:1122:3344:101::a]:42391","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":31412,"response_code":"200"}
{"msg":"path is \"/data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.722335768Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"dataset is oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.730697134Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"snapshot is oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322@8546765b-299a-4074-b332-a3f1ba0d9bb8","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.738412725Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322\", \"8546765b-299a-4074-b332-a3f1ba0d9bb8\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.738442826Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 92c00360-cfdb-42cd-9ddd-6d1f24359322-8546765b-299a-4074-b332-a3f1ba0d9bb8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.74838732Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"applying SMF actions for region 92c00360-cfdb-42cd-9ddd-6d1f24359322 running snapshot 8546765b-299a-4074-b332-a3f1ba0d9bb8 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.749034443Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.749084863Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322/snapshots/8546765b-299a-4074-b332-a3f1ba0d9bb8/run","method":"POST","req_id":"b91cc650-0251-48d5-9ebe-ad9ddd62b13a","remote_addr":"[fd00:1122:3344:101::a]:42391","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26776,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.757259638Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.757345399Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.757427069Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.757437449Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.757476389Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.757523149Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.75762218Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.7576826Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.757729571Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.757778921Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.76684163Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-92c00360-cfdb-42cd-9ddd-6d1f24359322-8546765b-299a-4074-b332-a3f1ba0d9bb8","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.767327143Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-92c00360-cfdb-42cd-9ddd-6d1f24359322-8546765b-299a-4074-b332-a3f1ba0d9bb8","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.776509843Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-92c00360-cfdb-42cd-9ddd-6d1f24359322-8546765b-299a-4074-b332-a3f1ba0d9bb8","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.797105705Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322/.zfs/snapshot/8546765b-299a-4074-b332-a3f1ba0d9bb8","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.797418897Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.797476137Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.797519577Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.797558598Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.797610128Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.801841111Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.809292741Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"region 92c00360-cfdb-42cd-9ddd-6d1f24359322 running snapshot 8546765b-299a-4074-b332-a3f1ba0d9bb8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:28.809304322Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.664210713Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49203"}
{"msg":"region d4b25bfe-ee64-4645-a42f-d440b640d9e6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.664332054Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.666749487Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.666769417Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions","method":"POST","req_id":"df5fe6ab-7948-4b52-b220-ddba83572600","remote_addr":"[fd00:1122:3344:101::a]:49203","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2466,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.676090458Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.676128838Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d4b25bfe-ee64-4645-a42f-d440b640d9e6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.743749686Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"region":"d4b25bfe-ee64-4645-a42f-d440b640d9e6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d4b25bfe-ee64-4645-a42f-d440b640d9e6\" \"--data\" \"/data/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.745634607Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"region":"d4b25bfe-ee64-4645-a42f-d440b640d9e6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.828176146Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions","method":"POST","req_id":"2f7fc143-bb4d-4406-a7c5-25899c81a8c9","remote_addr":"[fd00:1122:3344:101::a]:49203","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.86197442Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"region":"d4b25bfe-ee64-4645-a42f-d440b640d9e6","component":"worker"}
{"msg":"region d4b25bfe-ee64-4645-a42f-d440b640d9e6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.86200853Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.862123931Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.862783874Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.862885425Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.862984885Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.862993045Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.863046936Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.863108866Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.863509828Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.863568918Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.863625799Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.863672559Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.872798369Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d4b25bfe-ee64-4645-a42f-d440b640d9e6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.873237061Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d4b25bfe-ee64-4645-a42f-d440b640d9e6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.887211007Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.929198536Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d4b25bfe-ee64-4645-a42f-d440b640d9e6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.939598712Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.939882134Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.939962894Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.940018065Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.940080475Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.94652176Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:31.954430183Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:32.160578085Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions","method":"POST","req_id":"789ad15d-90e8-4c8f-a7ce-293c4c0de90d","remote_addr":"[fd00:1122:3344:101::a]:49203","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.926449538Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6","method":"GET","req_id":"309d1951-5c1e-448f-ae18-578963db1a75","remote_addr":"[fd00:1122:3344:101::a]:49203","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"path is \"/data/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.929211753Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"dataset is oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.940452794Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"region d4b25bfe-ee64-4645-a42f-d440b640d9e6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.948639549Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.94883931Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.94890413Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6","method":"DELETE","req_id":"c7e9e7c0-586a-44c9-94f7-cf1c2365dd34","remote_addr":"[fd00:1122:3344:101::a]:49203","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19722,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-d4b25bfe-ee64-4645-a42f-d440b640d9e6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.949961926Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.952086788Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6","method":"GET","req_id":"ea23ba7a-336e-421c-abad-f1379ffb754b","remote_addr":"[fd00:1122:3344:101::a]:49203","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":526,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.958339511Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.958422112Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.958505392Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.958513683Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.958567953Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.958622943Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c00360-cfdb-42cd-9ddd-6d1f24359322","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.958933085Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.959004685Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.959062886Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.959110156Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.965741892Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.96715557Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ce236cd3-1211-4ed1-b293-f36094cd8259/crucible/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:56.976582761Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"region":"d4b25bfe-ee64-4645-a42f-d440b640d9e6","component":"worker"}
{"msg":"region d4b25bfe-ee64-4645-a42f-d440b640d9e6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:57.027519338Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-16T03:55:57.088897512Z","hostname":"oxz_crucible_0ecd3fc1-1e37-4915-8617-89970ba6350c","pid":20603,"uri":"/crucible/0/regions/d4b25bfe-ee64-4645-a42f-d440b640d9e6","method":"GET","req_id":"c86ebeaa-4fdc-4776-8278-95dabef3cbb4","remote_addr":"[fd00:1122:3344:101::a]:49203","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":68,"response_code":"200"}