[ Aug 8 23:00:11 Enabled. ] [ Aug 8 23:00:11 Rereading configuration. ] [ Aug 8 23:00:13 Rereading configuration. ] [ Aug 8 23:00:15 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.211084214Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855} {"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.212349201Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.212363611Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.231484738Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:00:15.382927911Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.060702017Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43289"} {"msg":"region 00f7623a-eccf-4733-87f7-ece24618b0e3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.081803751Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.08524725Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.088715498Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions","method":"POST","req_id":"5c68b96d-e124-4c90-8b76-7ce5c57fa32e","remote_addr":"[fd00:1122:3344:101::a]:43289","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":12846,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.096905712Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.096932463Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"00f7623a-eccf-4733-87f7-ece24618b0e3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.16838519Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"00f7623a-eccf-4733-87f7-ece24618b0e3","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"00f7623a-eccf-4733-87f7-ece24618b0e3\" \"--data\" \"/data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.170297076Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"00f7623a-eccf-4733-87f7-ece24618b0e3","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.254629923Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions","method":"POST","req_id":"93326272-9774-4f83-813f-09e2569d40c1","remote_addr":"[fd00:1122:3344:101::a]:43289","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":82,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.318705587Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"00f7623a-eccf-4733-87f7-ece24618b0e3","component":"worker"} {"msg":"region 00f7623a-eccf-4733-87f7-ece24618b0e3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.318748848Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.318845319Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.319241635Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.326537286Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.341534166Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.346219171Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.354809651Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.356809009Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.35688297Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.356941211Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.360743244Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.364938922Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:01:35.848205835Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions","method":"POST","req_id":"59291fcd-7afc-4840-8909-18d1d58d860c","remote_addr":"[fd00:1122:3344:101::a]:43289","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.694107326Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33955"} {"msg":"region 45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.694301969Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.694873367Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.694885367Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions","method":"POST","req_id":"9f309107-c56d-4061-a8a2-897840e45f1a","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":626,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.704747504Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.704778805Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.763011737Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0\" \"--data\" \"/data/regions/45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.763064558Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.834752669Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","component":"worker"} {"msg":"region 45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.834795939Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.834888431Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.840896314Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.840976916Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.841055397Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.841064507Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.841105517Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.841169498Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.84126482Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.84132842Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.841377531Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.841425372Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.849292921Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.849706177Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.853204106Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.86069244Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.863608041Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/45d980d4-6e3e-4e0b-8255-e1a9aa7d49d0","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.863950226Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.864006477Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.864045927Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.864087808Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.867436085Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.872068359Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:44.933238383Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions","method":"POST","req_id":"cc8e4995-aa9d-4b5e-8b68-45742b1a12b6","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.895844685Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","method":"GET","req_id":"f6bffe69-1d8e-4c90-9903-db2baa520088","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":69,"response_code":"200"} {"msg":"path is \"/data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.901315311Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"dataset is oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.910732872Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"snapshot is oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3@40d9e99b-677d-4260-8266-4731a0533762","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.91912596Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3\", \"40d9e99b-677d-4260-8266-4731a0533762\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.91915375Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.929474074Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/00f7623a-eccf-4733-87f7-ece24618b0e3/snapshots/40d9e99b-677d-4260-8266-4731a0533762","method":"GET","req_id":"1dba9f6c-9d9e-450f-9a2c-8db7a7599f87","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":31300,"response_code":"200"} {"msg":"path is \"/data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.934088289Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"dataset is oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.943046983Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"snapshot is oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3@40d9e99b-677d-4260-8266-4731a0533762","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.952178521Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3\", \"40d9e99b-677d-4260-8266-4731a0533762\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.952207611Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 00f7623a-eccf-4733-87f7-ece24618b0e3-40d9e99b-677d-4260-8266-4731a0533762 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.961468791Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"applying SMF actions for region 00f7623a-eccf-4733-87f7-ece24618b0e3 running snapshot 40d9e99b-677d-4260-8266-4731a0533762 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.961642663Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.961687954Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/00f7623a-eccf-4733-87f7-ece24618b0e3/snapshots/40d9e99b-677d-4260-8266-4731a0533762/run","method":"POST","req_id":"d785cc9f-43b0-475b-a788-325ff18b2c8f","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27626,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962198351Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962278512Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962359443Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962366533Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962406424Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962452044Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962822069Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.96288176Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962928591Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.962974512Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.966516431Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-00f7623a-eccf-4733-87f7-ece24618b0e3-40d9e99b-677d-4260-8266-4731a0533762","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.96785737Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-00f7623a-eccf-4733-87f7-ece24618b0e3-40d9e99b-677d-4260-8266-4731a0533762","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.974082667Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-00f7623a-eccf-4733-87f7-ece24618b0e3-40d9e99b-677d-4260-8266-4731a0533762","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.984101416Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3/.zfs/snapshot/40d9e99b-677d-4260-8266-4731a0533762","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.98440094Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.984458571Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.984501342Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.984541553Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.984589193Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.988505968Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.993062021Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"region 00f7623a-eccf-4733-87f7-ece24618b0e3 running snapshot 40d9e99b-677d-4260-8266-4731a0533762 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:45.993075212Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"path is \"/data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:46.150346166Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"dataset is oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:46.159501924Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"snapshot is oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3@40d9e99b-677d-4260-8266-4731a0533762","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:46.167894761Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/00f7623a-eccf-4733-87f7-ece24618b0e3\", \"40d9e99b-677d-4260-8266-4731a0533762\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:46.167922531Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:46.177426544Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/00f7623a-eccf-4733-87f7-ece24618b0e3/snapshots/40d9e99b-677d-4260-8266-4731a0533762/run","method":"POST","req_id":"86ff3bca-7933-4745-98a5-122f9e7fab77","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27116,"response_code":"200"} {"msg":"region b18da655-4d78-46ae-8132-e72d7e4b19a6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.461649289Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.461850961Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.461874052Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions","method":"POST","req_id":"fd959f24-7ef2-4ccb-8963-0805170e2cf1","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":281,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.471709059Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.4717428Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b18da655-4d78-46ae-8132-e72d7e4b19a6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.543102775Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"b18da655-4d78-46ae-8132-e72d7e4b19a6","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b18da655-4d78-46ae-8132-e72d7e4b19a6\" \"--data\" \"/data/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.545064143Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"b18da655-4d78-46ae-8132-e72d7e4b19a6","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.609964218Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"b18da655-4d78-46ae-8132-e72d7e4b19a6","component":"worker"} {"msg":"region b18da655-4d78-46ae-8132-e72d7e4b19a6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.610007409Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.61009901Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.610736439Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.61081322Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.610894101Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.610902071Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.610944142Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.611008803Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.611404888Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.611469559Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.61151989Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.611567931Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.61226442Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions","method":"POST","req_id":"e0a617ba-0bfd-4551-9237-bd1905ff48fd","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.61582656Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b18da655-4d78-46ae-8132-e72d7e4b19a6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.616987776Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b18da655-4d78-46ae-8132-e72d7e4b19a6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.620409874Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.628236243Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b18da655-4d78-46ae-8132-e72d7e4b19a6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.631402478Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.63161483Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.631668971Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.631708452Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.631746492Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.636075743Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:06:49.641801132Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.706522406Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","method":"GET","req_id":"a2e6e477-bb97-47b2-b6a8-9685f49e462d","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":60,"response_code":"200"} {"msg":"path is \"/data/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.708741347Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"dataset is oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.718796347Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"region b18da655-4d78-46ae-8132-e72d7e4b19a6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.727321506Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.727546659Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.72760773Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","method":"DELETE","req_id":"8ff75e74-323b-4c55-97d9-34cf987d3c10","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18883,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-b18da655-4d78-46ae-8132-e72d7e4b19a6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.728725546Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.729854892Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","method":"GET","req_id":"26380047-efc7-4d6c-bed8-be4a359b47b5","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":21,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.73695542Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.737048882Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.737130333Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.737149423Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.737188654Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.737251915Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/00f7623a-eccf-4733-87f7-ece24618b0e3","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.73762822Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.737688771Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.737746972Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.737806763Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.742773152Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.74483273Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_3c3bcffd-6a11-43d2-a199-ac63d5e9ccb4/crucible/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.755413238Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"region":"b18da655-4d78-46ae-8132-e72d7e4b19a6","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.799090588Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","method":"GET","req_id":"3782a13b-4c9e-4507-befa-a6401693c66b","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":"200"} {"msg":"region b18da655-4d78-46ae-8132-e72d7e4b19a6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.813992386Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-08T23:07:19.905703035Z","hostname":"oxz_crucible_5ddf3bc2-cfa9-4c9a-84ed-68a469f199f1","pid":24855,"uri":"/crucible/0/regions/b18da655-4d78-46ae-8132-e72d7e4b19a6","method":"GET","req_id":"73d9b9fb-744a-4167-9e71-aa839bdabdbf","remote_addr":"[fd00:1122:3344:101::a]:33955","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25,"response_code":"200"}