[ Jul 25 05:48:05 Enabled. ]
[ Jul 25 05:48:05 Rereading configuration. ]
[ Jul 25 05:48:10 Rereading configuration. ]
[ Jul 25 05:48:13 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.737709793Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.749566105Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.749581955Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.777964519Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.937955559Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.63920817Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:51502"}
{"msg":"region 795a06de-b6d7-4454-b865-904ccae0f9e2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.656811238Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.664525681Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions","method":"POST","req_id":"1e5f7de9-dd49-4324-9cbc-27277af3d052","remote_addr":"[fd00:1122:3344:101::c]:51502","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":14453,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.666839146Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/795a06de-b6d7-4454-b865-904ccae0f9e2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.675395209Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/795a06de-b6d7-4454-b865-904ccae0f9e2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.67542932Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"795a06de-b6d7-4454-b865-904ccae0f9e2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/795a06de-b6d7-4454-b865-904ccae0f9e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.730730573Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"795a06de-b6d7-4454-b865-904ccae0f9e2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"795a06de-b6d7-4454-b865-904ccae0f9e2\" \"--data\" \"/data/regions/795a06de-b6d7-4454-b865-904ccae0f9e2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.732663179Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"795a06de-b6d7-4454-b865-904ccae0f9e2","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.84850391Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"795a06de-b6d7-4454-b865-904ccae0f9e2","component":"worker"}
{"msg":"region 795a06de-b6d7-4454-b865-904ccae0f9e2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.848597272Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.848776677Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-795a06de-b6d7-4454-b865-904ccae0f9e2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.849250928Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-795a06de-b6d7-4454-b865-904ccae0f9e2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.854838281Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.855062296Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions","method":"POST","req_id":"2bcd3a9b-4942-4f65-bfe8-e8288023195e","remote_addr":"[fd00:1122:3344:101::c]:51502","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.874123728Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-795a06de-b6d7-4454-b865-904ccae0f9e2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.883284806Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/795a06de-b6d7-4454-b865-904ccae0f9e2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.884084795Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.888718535Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.888773466Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.888818688Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.8952476Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.89946644Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.60822144Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions/795a06de-b6d7-4454-b865-904ccae0f9e2","method":"GET","req_id":"b8a85ea1-e122-401b-a28d-66ebe0b7166c","remote_addr":"[fd00:1122:3344:101::c]:51502","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":102,"response_code":"200"}
{"msg":"path is \"/data/regions/795a06de-b6d7-4454-b865-904ccae0f9e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.61201803Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"dataset is oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/795a06de-b6d7-4454-b865-904ccae0f9e2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.620603044Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"region 795a06de-b6d7-4454-b865-904ccae0f9e2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.628997373Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.629195828Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.62928912Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions/795a06de-b6d7-4454-b865-904ccae0f9e2","method":"DELETE","req_id":"6a7c9037-b79d-40f5-ab11-710414ed3a9b","remote_addr":"[fd00:1122:3344:101::c]:51502","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":18002,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-795a06de-b6d7-4454-b865-904ccae0f9e2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.630452388Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.632506867Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions/795a06de-b6d7-4454-b865-904ccae0f9e2","method":"GET","req_id":"84ad21a9-deec-46a4-89c3-eb632cceab5b","remote_addr":"[fd00:1122:3344:101::c]:51502","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.634688949Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/795a06de-b6d7-4454-b865-904ccae0f9e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.643347304Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"795a06de-b6d7-4454-b865-904ccae0f9e2","component":"worker"}
{"msg":"region 795a06de-b6d7-4454-b865-904ccae0f9e2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.696615909Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.735958233Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions/795a06de-b6d7-4454-b865-904ccae0f9e2","method":"GET","req_id":"9f2b5fa3-d39d-4a71-ba6d-c9b4179b10d8","remote_addr":"[fd00:1122:3344:101::c]:51502","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.25290132Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51169"}
{"msg":"region 6744bea5-2d71-4283-829d-903a2c16b105 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.253034283Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.256018194Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.256058485Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions","method":"POST","req_id":"afaa1017-21d3-4e69-8f10-45eb3af60fd6","remote_addr":"[fd00:1122:3344:101::a]:51169","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":3040,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/6744bea5-2d71-4283-829d-903a2c16b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.264142367Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/6744bea5-2d71-4283-829d-903a2c16b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.264174348Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6744bea5-2d71-4283-829d-903a2c16b105\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6744bea5-2d71-4283-829d-903a2c16b105\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.314987514Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"6744bea5-2d71-4283-829d-903a2c16b105","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6744bea5-2d71-4283-829d-903a2c16b105\" \"--data\" \"/data/regions/6744bea5-2d71-4283-829d-903a2c16b105\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.315053346Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"6744bea5-2d71-4283-829d-903a2c16b105","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.382663221Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"6744bea5-2d71-4283-829d-903a2c16b105","component":"worker"}
{"msg":"region 6744bea5-2d71-4283-829d-903a2c16b105 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.382703353Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.382792564Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6744bea5-2d71-4283-829d-903a2c16b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.384140787Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6744bea5-2d71-4283-829d-903a2c16b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.387255841Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.395453235Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6744bea5-2d71-4283-829d-903a2c16b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.398211831Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6744bea5-2d71-4283-829d-903a2c16b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.398473587Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.398532418Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.398573499Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.39861287Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.402011021Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.40660253Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.604504389Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions","method":"POST","req_id":"5a3b81dc-ee7b-4bbe-b55b-734ed7e473a8","remote_addr":"[fd00:1122:3344:101::a]:51169","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.875151102Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37490"}
{"msg":"region 347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.875350856Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.878244485Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.878259686Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions","method":"POST","req_id":"7ae65d04-ab32-4e1d-88d6-7de30b797928","remote_addr":"[fd00:1122:3344:101::a]:37490","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":3075,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.887735751Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.887764931Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.994442465Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2\" \"--data\" \"/data/regions/347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.994501766Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.195092499Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions","method":"POST","req_id":"09755dbc-7818-4f4f-9f62-a8ce78adf59a","remote_addr":"[fd00:1122:3344:101::a]:37490","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.24438538Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"region":"347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","component":"worker"}
{"msg":"region 347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.244429381Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.244561334Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.245579108Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.286448709Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.332096353Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.344331203Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/347d8b2d-3b3e-4674-9eb8-3ae8ec2301c2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.344553018Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.34461676Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.344665981Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.344713792Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.353685405Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.369778288Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.753921179Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions","method":"POST","req_id":"ba3dad4b-9d65-4243-a119-5baf578a1807","remote_addr":"[fd00:1122:3344:101::a]:37490","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":619,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.753254759Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45472"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.753338681Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions/6744bea5-2d71-4283-829d-903a2c16b105","method":"GET","req_id":"f36782fb-9930-43d3-9099-e1dc8f0c1c5c","remote_addr":"[fd00:1122:3344:101::a]:45472","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"path is \"/data/regions/6744bea5-2d71-4283-829d-903a2c16b105\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.753557536Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"dataset is oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/6744bea5-2d71-4283-829d-903a2c16b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.763354609Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"snapshot is oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/6744bea5-2d71-4283-829d-903a2c16b105@02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.774621177Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/6744bea5-2d71-4283-829d-903a2c16b105\", \"02c17e09-5830-4ee8-a416-a66468634fe2\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.774679368Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.788794373Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions/6744bea5-2d71-4283-829d-903a2c16b105/snapshots/02c17e09-5830-4ee8-a416-a66468634fe2","method":"GET","req_id":"c191beff-5616-4ff9-88e2-09c07fdc1322","remote_addr":"[fd00:1122:3344:101::a]:45472","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":35261,"response_code":"200"}
{"msg":"path is \"/data/regions/6744bea5-2d71-4283-829d-903a2c16b105\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.790137515Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"dataset is oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/6744bea5-2d71-4283-829d-903a2c16b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.801430033Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"snapshot is oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/6744bea5-2d71-4283-829d-903a2c16b105@02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.811036811Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_63a12260-6721-4f51-85d5-262aedc90628/crucible/regions/6744bea5-2d71-4283-829d-903a2c16b105\", \"02c17e09-5830-4ee8-a416-a66468634fe2\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.811078752Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 6744bea5-2d71-4283-829d-903a2c16b105-02c17e09-5830-4ee8-a416-a66468634fe2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.824113812Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}
{"msg":"applying SMF actions for region 6744bea5-2d71-4283-829d-903a2c16b105 running snapshot 02c17e09-5830-4ee8-a416-a66468634fe2 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.824297316Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.824346497Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"uri":"/crucible/0/regions/6744bea5-2d71-4283-829d-903a2c16b105/snapshots/02c17e09-5830-4ee8-a416-a66468634fe2/run","method":"POST","req_id":"754544c0-5a30-433b-a9f3-28907bad9393","remote_addr":"[fd00:1122:3344:101::a]:45472","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":35349,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-6744bea5-2d71-4283-829d-903a2c16b105-02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.827366529Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-6744bea5-2d71-4283-829d-903a2c16b105-02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.854108734Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-6744bea5-2d71-4283-829d-903a2c16b105-02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.889130546Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6744bea5-2d71-4283-829d-903a2c16b105/.zfs/snapshot/02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.889413693Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.889472674Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.889512985Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.889552526Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.889605807Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.902126684Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.908381923Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"worker"}
{"msg":"region 6744bea5-2d71-4283-829d-903a2c16b105 running snapshot 02c17e09-5830-4ee8-a416-a66468634fe2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.908412384Z","hostname":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","pid":28572,"component":"datafile"}