[ Aug  5 16:46:20 Enabled. ]
[ Aug  5 16:46:20 Rereading configuration. ]
[ Aug  5 16:46:24 Rereading configuration. ]
[ Aug  5 16:46:30 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:30.838232419Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:30.840501978Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:30.840511318Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:30.848057697Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:31.160156779Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.440344147Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:53903"}
{"msg":"region 5f17de4c-7cba-4094-a262-ebbf7dfe1010 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.462925502Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.463887699Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.467823296Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions","method":"POST","req_id":"c0ce99e9-79e7-4716-9fc9-402922448760","remote_addr":"[fd00:1122:3344:101::c]:53903","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":11705,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.474888766Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.474927057Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5f17de4c-7cba-4094-a262-ebbf7dfe1010\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.572756465Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"5f17de4c-7cba-4094-a262-ebbf7dfe1010","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5f17de4c-7cba-4094-a262-ebbf7dfe1010\" \"--data\" \"/data/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.575141376Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"5f17de4c-7cba-4094-a262-ebbf7dfe1010","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.773815504Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions","method":"POST","req_id":"86a2bd3b-1d2f-47b4-81ff-0f4937da78bc","remote_addr":"[fd00:1122:3344:101::c]:53903","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.831409136Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"5f17de4c-7cba-4094-a262-ebbf7dfe1010","component":"worker"}
{"msg":"region 5f17de4c-7cba-4094-a262-ebbf7dfe1010 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.831475947Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.83165969Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5f17de4c-7cba-4094-a262-ebbf7dfe1010","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.832062707Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5f17de4c-7cba-4094-a262-ebbf7dfe1010","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.837661763Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.849674217Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5f17de4c-7cba-4094-a262-ebbf7dfe1010","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.855990945Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.856730098Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.856815709Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.85685794Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.856904711Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.863500723Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:57.870630185Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.330142581Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions","method":"POST","req_id":"1663f63c-1d12-4aae-ae70-39fb111d9526","remote_addr":"[fd00:1122:3344:101::c]:53903","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":547,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.871718056Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010","method":"GET","req_id":"e0915122-f615-4cec-b134-d1f7734828d2","remote_addr":"[fd00:1122:3344:101::c]:53903","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.874417692Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"dataset is oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.883137071Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"region 5f17de4c-7cba-4094-a262-ebbf7dfe1010 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.892063843Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.892316308Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.892400569Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010","method":"DELETE","req_id":"10b2382a-a615-4219-8d23-4873f57e527f","remote_addr":"[fd00:1122:3344:101::c]:53903","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18017,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-5f17de4c-7cba-4094-a262-ebbf7dfe1010 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.893573439Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.896419618Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010","method":"GET","req_id":"6f6b8f27-0ea8-4893-8008-9baf4f610563","remote_addr":"[fd00:1122:3344:101::c]:53903","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":573,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.90066929Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.910819343Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"5f17de4c-7cba-4094-a262-ebbf7dfe1010","component":"worker"}
{"msg":"region 5f17de4c-7cba-4094-a262-ebbf7dfe1010 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:58.974367217Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:46:59.021123694Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/5f17de4c-7cba-4094-a262-ebbf7dfe1010","method":"GET","req_id":"385a3d80-022e-4f63-a758-06161bc608ef","remote_addr":"[fd00:1122:3344:101::c]:53903","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.353931788Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43102"}
{"msg":"region 8985df33-230a-4110-96c8-6d844525886d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.354169402Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.354359865Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions","method":"POST","req_id":"a35fc422-a447-4c82-830e-0d0e122ad40c","remote_addr":"[fd00:1122:3344:101::a]:43102","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":229,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.354377656Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/8985df33-230a-4110-96c8-6d844525886d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.362953102Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/8985df33-230a-4110-96c8-6d844525886d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.362982852Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8985df33-230a-4110-96c8-6d844525886d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8985df33-230a-4110-96c8-6d844525886d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.433151769Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"8985df33-230a-4110-96c8-6d844525886d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8985df33-230a-4110-96c8-6d844525886d\" \"--data\" \"/data/regions/8985df33-230a-4110-96c8-6d844525886d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.43322086Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"8985df33-230a-4110-96c8-6d844525886d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.511925132Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"8985df33-230a-4110-96c8-6d844525886d","component":"worker"}
{"msg":"region 8985df33-230a-4110-96c8-6d844525886d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.511977463Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.512060674Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8985df33-230a-4110-96c8-6d844525886d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.513486849Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8985df33-230a-4110-96c8-6d844525886d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.516715614Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.51703683Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions","method":"POST","req_id":"d0307763-dd90-45ce-8a5c-c7a898ba4b19","remote_addr":"[fd00:1122:3344:101::a]:43102","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.525632806Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8985df33-230a-4110-96c8-6d844525886d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.530184144Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8985df33-230a-4110-96c8-6d844525886d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.530491589Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.53054783Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.530595991Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.530644351Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.534372635Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:47:46.538727789Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.188914255Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34135"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.189128439Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/8985df33-230a-4110-96c8-6d844525886d","method":"GET","req_id":"4c7d6595-c13e-4616-a009-f399ec4e4633","remote_addr":"[fd00:1122:3344:101::a]:34135","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"path is \"/data/regions/8985df33-230a-4110-96c8-6d844525886d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.193254279Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"dataset is oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/8985df33-230a-4110-96c8-6d844525886d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.202276093Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"snapshot is oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/8985df33-230a-4110-96c8-6d844525886d@11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.211494241Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/8985df33-230a-4110-96c8-6d844525886d\", \"11dc1e70-0522-4d3f-9b48-e43c5e60b88f\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.211569292Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.222254174Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/8985df33-230a-4110-96c8-6d844525886d/snapshots/11dc1e70-0522-4d3f-9b48-e43c5e60b88f","method":"GET","req_id":"dceb76cf-bcd0-4c3d-9ed9-1a5483d8e3a7","remote_addr":"[fd00:1122:3344:101::a]:34135","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":33026,"response_code":"200"}
{"msg":"path is \"/data/regions/8985df33-230a-4110-96c8-6d844525886d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.222429027Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"dataset is oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/8985df33-230a-4110-96c8-6d844525886d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.230286741Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"snapshot is oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/8985df33-230a-4110-96c8-6d844525886d@11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.238983739Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/8985df33-230a-4110-96c8-6d844525886d\", \"11dc1e70-0522-4d3f-9b48-e43c5e60b88f\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.2390171Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 8985df33-230a-4110-96c8-6d844525886d-11dc1e70-0522-4d3f-9b48-e43c5e60b88f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.247697028Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"applying SMF actions for region 8985df33-230a-4110-96c8-6d844525886d running snapshot 11dc1e70-0522-4d3f-9b48-e43c5e60b88f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.248002543Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.248023983Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/8985df33-230a-4110-96c8-6d844525886d/snapshots/11dc1e70-0522-4d3f-9b48-e43c5e60b88f/run","method":"POST","req_id":"f2d8ae0c-8cf8-4efa-aec1-b9fa02785c2e","remote_addr":"[fd00:1122:3344:101::a]:34135","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25643,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-8985df33-230a-4110-96c8-6d844525886d-11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.254150398Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-8985df33-230a-4110-96c8-6d844525886d-11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.279747984Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-8985df33-230a-4110-96c8-6d844525886d-11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.29768455Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8985df33-230a-4110-96c8-6d844525886d/.zfs/snapshot/11dc1e70-0522-4d3f-9b48-e43c5e60b88f","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.298023606Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.298077327Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.298118678Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.298158178Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.298206119Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.301989034Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.306606803Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"region 8985df33-230a-4110-96c8-6d844525886d running snapshot 11dc1e70-0522-4d3f-9b48-e43c5e60b88f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:13.306617483Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.094095287Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49668"}
{"msg":"region 0da0229b-6cc9-4f9c-80c9-32acb80218b5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.094307391Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.094502314Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions","method":"POST","req_id":"b6bdb469-9ee7-48ec-8dc7-eb50c356e0ea","remote_addr":"[fd00:1122:3344:101::a]:49668","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":248,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.094521864Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.105075144Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.105104735Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0da0229b-6cc9-4f9c-80c9-32acb80218b5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.16758743Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"0da0229b-6cc9-4f9c-80c9-32acb80218b5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0da0229b-6cc9-4f9c-80c9-32acb80218b5\" \"--data\" \"/data/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.167644611Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"0da0229b-6cc9-4f9c-80c9-32acb80218b5","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.234878258Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"0da0229b-6cc9-4f9c-80c9-32acb80218b5","component":"worker"}
{"msg":"region 0da0229b-6cc9-4f9c-80c9-32acb80218b5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.234917789Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.23503029Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0da0229b-6cc9-4f9c-80c9-32acb80218b5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.236135059Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0da0229b-6cc9-4f9c-80c9-32acb80218b5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.238898836Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.253454375Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0da0229b-6cc9-4f9c-80c9-32acb80218b5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.257978972Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.258245066Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.258297627Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.258345668Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.258387489Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.262127293Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.273373775Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:16.433931493Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions","method":"POST","req_id":"f5532f54-5f9c-423f-af6e-c097663c5ee7","remote_addr":"[fd00:1122:3344:101::a]:49668","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.45734967Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5","method":"GET","req_id":"cd30021e-b5fd-44ff-818f-7e426c8731c1","remote_addr":"[fd00:1122:3344:101::a]:49668","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"path is \"/data/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.460108887Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"dataset is oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.471970369Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"region 0da0229b-6cc9-4f9c-80c9-32acb80218b5 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.479980046Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.480192079Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.48024019Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5","method":"DELETE","req_id":"6acb11f1-a8a7-49d3-8f54-8cba1855aa40","remote_addr":"[fd00:1122:3344:101::a]:49668","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20153,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-0da0229b-6cc9-4f9c-80c9-32acb80218b5 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.481840847Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.482998797Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5","method":"GET","req_id":"a6df7c34-e048-428d-b0bf-7b2e2773628d","remote_addr":"[fd00:1122:3344:101::a]:49668","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.493897353Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crucible/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.505342068Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"region":"0da0229b-6cc9-4f9c-80c9-32acb80218b5","component":"worker"}
{"msg":"region 0da0229b-6cc9-4f9c-80c9-32acb80218b5 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.586900089Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-05T16:53:40.634313427Z","hostname":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","pid":29610,"uri":"/crucible/0/regions/0da0229b-6cc9-4f9c-80c9-32acb80218b5","method":"GET","req_id":"e4cc8cbf-114c-4970-bbf1-0660169eac56","remote_addr":"[fd00:1122:3344:101::a]:49668","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":689,"response_code":"200"}