[ Oct 24 18:54:35 Enabled. ]
[ Oct 24 18:54:35 Rereading configuration. ]
[ Oct 24 18:54:39 Rereading configuration. ]
[ Oct 24 18:54:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.209304221Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.213642757Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.213656148Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.243172803Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.407334643Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.086529049Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:35402"}
{"msg":"region 47ff7776-68f8-454f-ae4f-873341ce30f4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.115237611Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.120461902Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions","method":"POST","req_id":"b769113d-42e8-49ac-a871-55b57d80e063","remote_addr":"[fd00:1122:3344:101::c]:35402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":13607,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.123018111Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.132545738Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.132599609Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"47ff7776-68f8-454f-ae4f-873341ce30f4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/47ff7776-68f8-454f-ae4f-873341ce30f4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.199009763Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"region":"47ff7776-68f8-454f-ae4f-873341ce30f4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"47ff7776-68f8-454f-ae4f-873341ce30f4\" \"--data\" \"/data/regions/47ff7776-68f8-454f-ae4f-873341ce30f4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.201053114Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"region":"47ff7776-68f8-454f-ae4f-873341ce30f4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.321590122Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"region":"47ff7776-68f8-454f-ae4f-873341ce30f4","component":"worker"}
{"msg":"region 47ff7776-68f8-454f-ae4f-873341ce30f4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.321680674Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.321876446Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-47ff7776-68f8-454f-ae4f-873341ce30f4","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.324166782Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-47ff7776-68f8-454f-ae4f-873341ce30f4","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.327648895Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.338107427Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-47ff7776-68f8-454f-ae4f-873341ce30f4","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.341878185Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.348416855Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.348490537Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.348538117Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.348572238Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.353305361Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.359485576Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.394914572Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions","method":"POST","req_id":"2349cf66-c9ff-4f74-ba77-970fa20099e7","remote_addr":"[fd00:1122:3344:101::c]:35402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.274475241Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","method":"GET","req_id":"e5daa045-1416-42c1-b2d0-5bd2b3687ba9","remote_addr":"[fd00:1122:3344:101::c]:35402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"path is \"/data/regions/47ff7776-68f8-454f-ae4f-873341ce30f4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.27834079Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"dataset is oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.28935424Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"region 47ff7776-68f8-454f-ae4f-873341ce30f4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.298872037Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.29908927Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.299172301Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","method":"DELETE","req_id":"5c578f5a-263e-4a87-8cef-46abff5ebc1b","remote_addr":"[fd00:1122:3344:101::c]:35402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20863,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-47ff7776-68f8-454f-ae4f-873341ce30f4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.300221787Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.302475502Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","method":"GET","req_id":"ab28b0ff-473e-4d44-be3e-f7a00faada03","remote_addr":"[fd00:1122:3344:101::c]:35402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.305688532Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/47ff7776-68f8-454f-ae4f-873341ce30f4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.316773903Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"region":"47ff7776-68f8-454f-ae4f-873341ce30f4","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.3925016Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","method":"GET","req_id":"ed247339-983e-4f4c-aa40-04e5a0e45814","remote_addr":"[fd00:1122:3344:101::c]:35402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":111,"response_code":"200"}
{"msg":"region 47ff7776-68f8-454f-ae4f-873341ce30f4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.392597671Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.544548474Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/47ff7776-68f8-454f-ae4f-873341ce30f4","method":"GET","req_id":"ca7e6697-645f-4bc5-8403-328088df3f42","remote_addr":"[fd00:1122:3344:101::c]:35402","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.360848419Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37126"}
{"msg":"region 4a16f78d-5a52-49e5-8ea0-6128a8b7b7da state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.361015464Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.36118582Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions","method":"POST","req_id":"aa7c749d-1a46-484b-ab82-9bdf9107be55","remote_addr":"[fd00:1122:3344:101::a]:37126","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":212,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.361209418Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.370704002Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.370741193Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.42248501Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"region":"4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\" \"--data\" \"/data/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.422547141Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"region":"4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.504288801Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions","method":"POST","req_id":"e9650360-e7e2-42ce-8467-b4b5dd50e37d","remote_addr":"[fd00:1122:3344:101::a]:37126","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.510770031Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"region":"4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","component":"worker"}
{"msg":"region 4a16f78d-5a52-49e5-8ea0-6128a8b7b7da state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.510793692Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.510917454Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.512350916Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.515435393Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.522503972Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.526406252Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.526728877Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.526780528Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.526818879Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.526858639Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.530255632Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:01.534764401Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:56:02.213209658Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions","method":"POST","req_id":"ba48aee2-75e1-4212-8145-6767bcdfc24d","remote_addr":"[fd00:1122:3344:101::a]:37126","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.119456976Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:65195"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.119608459Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","method":"GET","req_id":"20186901-36b1-459a-906a-8eae66aac7fd","remote_addr":"[fd00:1122:3344:101::a]:65195","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"path is \"/data/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.123312906Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"dataset is oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.132357486Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"snapshot is oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da@63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.141612078Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\", \"63eefcdc-6ccb-408c-aa68-17bee2851ce7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.141651459Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.154922833Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da/snapshots/63eefcdc-6ccb-408c-aa68-17bee2851ce7","method":"GET","req_id":"9c28093f-6237-42e9-b4c5-b4cfce4ab0ca","remote_addr":"[fd00:1122:3344:101::a]:65195","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":31608,"response_code":"200"}
{"msg":"path is \"/data/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.158428097Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"dataset is oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.166986039Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"snapshot is oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da@63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.174604227Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\", \"63eefcdc-6ccb-408c-aa68-17bee2851ce7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.174629147Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 4a16f78d-5a52-49e5-8ea0-6128a8b7b7da-63eefcdc-6ccb-408c-aa68-17bee2851ce7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.183417382Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"applying SMF actions for region 4a16f78d-5a52-49e5-8ea0-6128a8b7b7da running snapshot 63eefcdc-6ccb-408c-aa68-17bee2851ce7 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.184054632Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.184097279Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da/snapshots/63eefcdc-6ccb-408c-aa68-17bee2851ce7/run","method":"POST","req_id":"44a8bca7-6dbe-4278-8cc2-bfec2ef1c6f5","remote_addr":"[fd00:1122:3344:101::a]:65195","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25737,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-4a16f78d-5a52-49e5-8ea0-6128a8b7b7da-63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.192552643Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-4a16f78d-5a52-49e5-8ea0-6128a8b7b7da-63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.200466255Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-4a16f78d-5a52-49e5-8ea0-6128a8b7b7da-63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.218658016Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da/.zfs/snapshot/63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.21895031Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.219001981Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.219042562Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.219083432Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.219133173Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.226393575Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.233792759Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"worker"}
{"msg":"region 4a16f78d-5a52-49e5-8ea0-6128a8b7b7da running snapshot 63eefcdc-6ccb-408c-aa68-17bee2851ce7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.233812839Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"path is \"/data/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.466644788Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"dataset is oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.476336078Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"datafile"}
{"msg":"snapshot is oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da@63eefcdc-6ccb-408c-aa68-17bee2851ce7","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.48493817Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_62e8a5f9-9c11-40b3-8243-b440c8ecad6f/crucible/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da\", \"63eefcdc-6ccb-408c-aa68-17bee2851ce7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.484965091Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:59:56.493867468Z","hostname":"oxz_crucible_fc09e276-dc62-4ef0-a414-a788a80131ad","pid":24019,"uri":"/crucible/0/regions/4a16f78d-5a52-49e5-8ea0-6128a8b7b7da/snapshots/63eefcdc-6ccb-408c-aa68-17bee2851ce7/run","method":"POST","req_id":"629fb9a9-04f8-4d39-840f-5cdc34f40878","remote_addr":"[fd00:1122:3344:101::a]:65195","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27263,"response_code":"200"}