[ Jul 10 23:01:41 Enabled. ]
[ Jul 10 23:01:41 Rereading configuration. ]
[ Jul 10 23:01:42 Rereading configuration. ]
[ Jul 10 23:01:44 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:01:45.047858642Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:01:45.052123624Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:01:45.052142675Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:01:45.074394627Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:01:45.203674989Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.642738905Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43807"}
{"msg":"region be092782-558f-47b3-b90b-d3d52191ef49 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.647945048Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.65309534Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.65311197Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions","method":"POST","req_id":"b6619de0-4a66-4b3f-9381-11ce81911a54","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":5336,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/be092782-558f-47b3-b90b-d3d52191ef49","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.665463928Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/be092782-558f-47b3-b90b-d3d52191ef49","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.665497079Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"be092782-558f-47b3-b90b-d3d52191ef49\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/be092782-558f-47b3-b90b-d3d52191ef49\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.729992617Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"be092782-558f-47b3-b90b-d3d52191ef49","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"be092782-558f-47b3-b90b-d3d52191ef49\" \"--data\" \"/data/regions/be092782-558f-47b3-b90b-d3d52191ef49\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.730069218Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"be092782-558f-47b3-b90b-d3d52191ef49","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.838029668Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"be092782-558f-47b3-b90b-d3d52191ef49","component":"worker"}
{"msg":"region be092782-558f-47b3-b90b-d3d52191ef49 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.838087599Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.838169911Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-be092782-558f-47b3-b90b-d3d52191ef49","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.838610741Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-be092782-558f-47b3-b90b-d3d52191ef49","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.843891915Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.855349793Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-be092782-558f-47b3-b90b-d3d52191ef49","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.858124124Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/be092782-558f-47b3-b90b-d3d52191ef49","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.85886325Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.860861343Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.860918174Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.860957025Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.867798203Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.874154251Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:03.896161048Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions","method":"POST","req_id":"380995a6-2a3d-4653-8f95-77dc92566de8","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.869879601Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions/be092782-558f-47b3-b90b-d3d52191ef49","method":"GET","req_id":"965d36bb-3c29-4de0-82ff-ff3b15e9d5bc","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"path is \"/data/regions/be092782-558f-47b3-b90b-d3d52191ef49\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.873240454Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"dataset is oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/be092782-558f-47b3-b90b-d3d52191ef49","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.88228782Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"region be092782-558f-47b3-b90b-d3d52191ef49 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.891180163Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.891384587Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.89147592Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions/be092782-558f-47b3-b90b-d3d52191ef49","method":"DELETE","req_id":"54b575d3-9fe0-463c-89c9-9a291173a778","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18275,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-be092782-558f-47b3-b90b-d3d52191ef49 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.892642925Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.89426284Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions/be092782-558f-47b3-b90b-d3d52191ef49","method":"GET","req_id":"51239b8a-ce18-4ad9-8357-7003d1b3324e","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.897282405Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/be092782-558f-47b3-b90b-d3d52191ef49\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.90673422Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"be092782-558f-47b3-b90b-d3d52191ef49","component":"worker"}
{"msg":"region be092782-558f-47b3-b90b-d3d52191ef49 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:04.972300171Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:05.036268208Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions/be092782-558f-47b3-b90b-d3d52191ef49","method":"GET","req_id":"800a3ce1-5438-4982-aca8-5c104914aa4d","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"region a8a25c6f-c21a-469d-91b6-bf6fdbca5c33 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.692734088Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.694948426Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.694970346Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions","method":"POST","req_id":"449c7184-72f8-4cb3-bc25-aad8106d2fb4","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":2993,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.704315969Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.70438502Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a8a25c6f-c21a-469d-91b6-bf6fdbca5c33\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.78096638Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a8a25c6f-c21a-469d-91b6-bf6fdbca5c33\" \"--data\" \"/data/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.783060036Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.839005058Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions","method":"POST","req_id":"93de4e60-7cce-44d2-b4c2-d4a7d4ba91d1","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.848090725Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","component":"worker"}
{"msg":"region a8a25c6f-c21a-469d-91b6-bf6fdbca5c33 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.848152356Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.84832356Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.84968751Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.852995711Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.860678078Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.864156073Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.864359638Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.864407189Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.86444644Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.86449587Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.86860738Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:06.873154058Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:02:07.375183679Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions","method":"POST","req_id":"f23c563a-9732-43c4-9f36-31d55e094fe0","remote_addr":"[fd00:1122:3344:101::a]:43807","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.06823566Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61224"}
{"msg":"region 07603ca6-66fd-4954-82a9-a4fb3b6298eb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.068376153Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.069094159Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.06911973Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions","method":"POST","req_id":"e737ca91-ef76-4e4b-b7b0-5cfb264999e0","remote_addr":"[fd00:1122:3344:101::a]:61224","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":789,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/07603ca6-66fd-4954-82a9-a4fb3b6298eb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.079137056Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/07603ca6-66fd-4954-82a9-a4fb3b6298eb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.079169857Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"07603ca6-66fd-4954-82a9-a4fb3b6298eb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/07603ca6-66fd-4954-82a9-a4fb3b6298eb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.164485746Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"07603ca6-66fd-4954-82a9-a4fb3b6298eb","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"07603ca6-66fd-4954-82a9-a4fb3b6298eb\" \"--data\" \"/data/regions/07603ca6-66fd-4954-82a9-a4fb3b6298eb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.164550128Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"07603ca6-66fd-4954-82a9-a4fb3b6298eb","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.249215513Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"region":"07603ca6-66fd-4954-82a9-a4fb3b6298eb","component":"worker"}
{"msg":"region 07603ca6-66fd-4954-82a9-a4fb3b6298eb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.249268684Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.249354176Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-07603ca6-66fd-4954-82a9-a4fb3b6298eb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.250378498Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-07603ca6-66fd-4954-82a9-a4fb3b6298eb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.259289381Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.273499859Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-07603ca6-66fd-4954-82a9-a4fb3b6298eb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.277263231Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/07603ca6-66fd-4954-82a9-a4fb3b6298eb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.277464115Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.277519006Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.277562777Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.277603368Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.282580476Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.289268421Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:25.34922517Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions","method":"POST","req_id":"5fbc9b8e-81d6-4974-bc13-0287335d3314","remote_addr":"[fd00:1122:3344:101::a]:61224","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.830621426Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:36518"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.831370773Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","method":"GET","req_id":"143ba143-ec6e-41dc-9094-4fe472ae3535","remote_addr":"[fd00:1122:3344:101::a]:36518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":707,"response_code":"200"}
{"msg":"path is \"/data/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.831597208Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"dataset is oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.841494372Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"snapshot is oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33@9afadc7e-5ad8-44de-833e-b82e207e996e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.850684861Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33\", \"9afadc7e-5ad8-44de-833e-b82e207e996e\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.850717542Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.862385455Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33/snapshots/9afadc7e-5ad8-44de-833e-b82e207e996e","method":"GET","req_id":"8d7807a2-485c-4469-8035-f1e4be045827","remote_addr":"[fd00:1122:3344:101::a]:36518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":30831,"response_code":"200"}
{"msg":"path is \"/data/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.862577219Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"dataset is oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.871183555Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"snapshot is oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33@9afadc7e-5ad8-44de-833e-b82e207e996e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.87968876Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_68327ee5-4aec-4399-b8bb-563ba202e23d/crucible/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33\", \"9afadc7e-5ad8-44de-833e-b82e207e996e\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.87971166Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot a8a25c6f-c21a-469d-91b6-bf6fdbca5c33-9afadc7e-5ad8-44de-833e-b82e207e996e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.891652999Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}
{"msg":"applying SMF actions for region a8a25c6f-c21a-469d-91b6-bf6fdbca5c33 running snapshot 9afadc7e-5ad8-44de-833e-b82e207e996e (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.891812563Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.891862774Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"uri":"/crucible/0/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33/snapshots/9afadc7e-5ad8-44de-833e-b82e207e996e/run","method":"POST","req_id":"ce2e1cc4-4458-4adc-b811-5407ab2bc50f","remote_addr":"[fd00:1122:3344:101::a]:36518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29311,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-a8a25c6f-c21a-469d-91b6-bf6fdbca5c33-9afadc7e-5ad8-44de-833e-b82e207e996e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.894499291Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a8a25c6f-c21a-469d-91b6-bf6fdbca5c33-9afadc7e-5ad8-44de-833e-b82e207e996e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.897483765Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a8a25c6f-c21a-469d-91b6-bf6fdbca5c33-9afadc7e-5ad8-44de-833e-b82e207e996e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.90740416Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a8a25c6f-c21a-469d-91b6-bf6fdbca5c33/.zfs/snapshot/9afadc7e-5ad8-44de-833e-b82e207e996e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.907621785Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.907675497Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.907720767Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.907766058Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.907817009Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.911337226Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.915576528Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"worker"}
{"msg":"region a8a25c6f-c21a-469d-91b6-bf6fdbca5c33 running snapshot 9afadc7e-5ad8-44de-833e-b82e207e996e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T23:07:26.915586108Z","hostname":"oxz_crucible_9009234a-d43b-4467-bb7e-5aa69b874229","pid":29820,"component":"datafile"}