[ Jul 10 20:02:44 Enabled. ]
[ Jul 10 20:02:44 Rereading configuration. ]
[ Jul 10 20:02:46 Rereading configuration. ]
[ Jul 10 20:02:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.123090443Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.124772287Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.124789477Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.165436073Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.324006178Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.536745194Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:50205"}
{"msg":"region 54882e6d-646e-4597-bc2b-a9a838f0f21c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.555744021Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.561678721Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions","method":"POST","req_id":"de32ae94-bd64-462b-80bf-e54fd5547453","remote_addr":"[fd00:1122:3344:101::c]:50205","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14240,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.562226945Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.575551326Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.575590026Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"54882e6d-646e-4597-bc2b-a9a838f0f21c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.668196344Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"54882e6d-646e-4597-bc2b-a9a838f0f21c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"54882e6d-646e-4597-bc2b-a9a838f0f21c\" \"--data\" \"/data/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.670056019Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"54882e6d-646e-4597-bc2b-a9a838f0f21c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.875924854Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"54882e6d-646e-4597-bc2b-a9a838f0f21c","component":"worker"}
{"msg":"region 54882e6d-646e-4597-bc2b-a9a838f0f21c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.876150906Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.876380618Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-54882e6d-646e-4597-bc2b-a9a838f0f21c","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.876783872Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-54882e6d-646e-4597-bc2b-a9a838f0f21c","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.896003801Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.926315012Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions","method":"POST","req_id":"24bb26e9-6899-4060-a6ce-9b854ed0e114","remote_addr":"[fd00:1122:3344:101::c]:50205","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":93,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.94178371Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-54882e6d-646e-4597-bc2b-a9a838f0f21c","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.9586743Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.959592188Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.963583911Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.963654852Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.963693842Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.996792066Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.018854679Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.306426179Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","method":"GET","req_id":"c1f1dede-6ece-469c-8481-75698ca2b3f0","remote_addr":"[fd00:1122:3344:101::c]:50205","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":93,"response_code":"200"}
{"msg":"path is \"/data/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.308405426Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"dataset is oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.317606142Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"region 54882e6d-646e-4597-bc2b-a9a838f0f21c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.327447503Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.327603105Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.327666135Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","method":"DELETE","req_id":"942029e9-853b-44bb-a7ad-5d8ea045315e","remote_addr":"[fd00:1122:3344:101::c]:50205","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19318,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-54882e6d-646e-4597-bc2b-a9a838f0f21c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.328688794Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.329711752Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","method":"GET","req_id":"eb531e33-bb50-4ccf-8b27-975ae93c5e1c","remote_addr":"[fd00:1122:3344:101::c]:50205","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.348379697Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.358148278Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"54882e6d-646e-4597-bc2b-a9a838f0f21c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.411319669Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","method":"GET","req_id":"46e3f915-a9af-49d9-a3d4-86d021468d43","remote_addr":"[fd00:1122:3344:101::c]:50205","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region 54882e6d-646e-4597-bc2b-a9a838f0f21c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.435063035Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.478546876Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions/54882e6d-646e-4597-bc2b-a9a838f0f21c","method":"GET","req_id":"a3abe02d-5a33-4829-af32-47ab2cd6823e","remote_addr":"[fd00:1122:3344:101::c]:50205","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.384559127Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33501"}
{"msg":"region 106fca46-fef1-4b71-bb4b-7b85d6deea47 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.384678288Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.387041128Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions","method":"POST","req_id":"dae7fc18-6cb9-429b-aa29-e3af1d8154b0","remote_addr":"[fd00:1122:3344:101::a]:33501","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":2409,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.387063998Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.395337187Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.395372477Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"106fca46-fef1-4b71-bb4b-7b85d6deea47\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.458262718Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"106fca46-fef1-4b71-bb4b-7b85d6deea47","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"106fca46-fef1-4b71-bb4b-7b85d6deea47\" \"--data\" \"/data/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.460188754Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"106fca46-fef1-4b71-bb4b-7b85d6deea47","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.565505447Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"106fca46-fef1-4b71-bb4b-7b85d6deea47","component":"worker"}
{"msg":"region 106fca46-fef1-4b71-bb4b-7b85d6deea47 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.565541127Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.565650878Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-106fca46-fef1-4b71-bb4b-7b85d6deea47","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.567259491Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-106fca46-fef1-4b71-bb4b-7b85d6deea47","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.573751505Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.585353851Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-106fca46-fef1-4b71-bb4b-7b85d6deea47","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.591835205Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.592140018Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.592237318Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.592282269Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.592323609Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.599412078Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.604062296Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:04:01.716939472Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions","method":"POST","req_id":"74974178-2b87-4900-858d-4ca3e326a149","remote_addr":"[fd00:1122:3344:101::a]:33501","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.298083065Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39401"}
{"msg":"region 43bf9777-3115-43da-80e7-f772951318fd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.298234966Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.298925722Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions","method":"POST","req_id":"92bc35ea-7e5d-4440-9d04-2d408c883473","remote_addr":"[fd00:1122:3344:101::a]:39401","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":813,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.298942482Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/43bf9777-3115-43da-80e7-f772951318fd","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.308654692Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/43bf9777-3115-43da-80e7-f772951318fd","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.308695603Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"43bf9777-3115-43da-80e7-f772951318fd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/43bf9777-3115-43da-80e7-f772951318fd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.36143381Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"43bf9777-3115-43da-80e7-f772951318fd","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"43bf9777-3115-43da-80e7-f772951318fd\" \"--data\" \"/data/regions/43bf9777-3115-43da-80e7-f772951318fd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.36148928Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"43bf9777-3115-43da-80e7-f772951318fd","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.427166675Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"region":"43bf9777-3115-43da-80e7-f772951318fd","component":"worker"}
{"msg":"region 43bf9777-3115-43da-80e7-f772951318fd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.427226925Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.427332426Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-43bf9777-3115-43da-80e7-f772951318fd","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.434793628Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-43bf9777-3115-43da-80e7-f772951318fd","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.441286632Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.450191985Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-43bf9777-3115-43da-80e7-f772951318fd","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.45316504Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/43bf9777-3115-43da-80e7-f772951318fd","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.453369592Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.453419592Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.453460863Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.453497963Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.456937521Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.460927694Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.602980612Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions","method":"POST","req_id":"9be30df8-f490-431d-bc4e-04ef039f8b2a","remote_addr":"[fd00:1122:3344:101::a]:39401","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.482375299Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62942"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.4824883Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47","method":"GET","req_id":"e9b3a38c-8cbd-4cbd-a353-519395d2a645","remote_addr":"[fd00:1122:3344:101::a]:62942","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.482628821Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"dataset is oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.491116602Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"snapshot is oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47@9e302fab-288d-4912-8d43-d6946fbfde66","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.49935089Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47\", \"9e302fab-288d-4912-8d43-d6946fbfde66\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.49938349Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.510802105Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47/snapshots/9e302fab-288d-4912-8d43-d6946fbfde66","method":"GET","req_id":"48eb40b6-b764-4ef9-846d-cd924682d317","remote_addr":"[fd00:1122:3344:101::a]:62942","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28177,"response_code":"200"}
{"msg":"path is \"/data/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.510951066Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"dataset is oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.520756028Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"snapshot is oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47@9e302fab-288d-4912-8d43-d6946fbfde66","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.529596871Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_9c3571c0-266e-418f-b268-1185bbcb3a3c/crucible/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47\", \"9e302fab-288d-4912-8d43-d6946fbfde66\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.529652431Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 106fca46-fef1-4b71-bb4b-7b85d6deea47-9e302fab-288d-4912-8d43-d6946fbfde66 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.541717251Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}
{"msg":"applying SMF actions for region 106fca46-fef1-4b71-bb4b-7b85d6deea47 running snapshot 9e302fab-288d-4912-8d43-d6946fbfde66 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.541920623Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.541938023Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"uri":"/crucible/0/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47/snapshots/9e302fab-288d-4912-8d43-d6946fbfde66/run","method":"POST","req_id":"3418ec64-59a0-4829-9a0c-1ad1f84df4ae","remote_addr":"[fd00:1122:3344:101::a]:62942","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":31010,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-106fca46-fef1-4b71-bb4b-7b85d6deea47-9e302fab-288d-4912-8d43-d6946fbfde66","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.544722156Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-106fca46-fef1-4b71-bb4b-7b85d6deea47-9e302fab-288d-4912-8d43-d6946fbfde66","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.547835662Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-106fca46-fef1-4b71-bb4b-7b85d6deea47-9e302fab-288d-4912-8d43-d6946fbfde66","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.557680954Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/106fca46-fef1-4b71-bb4b-7b85d6deea47/.zfs/snapshot/9e302fab-288d-4912-8d43-d6946fbfde66","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.557900225Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.557953456Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.557993496Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.558032136Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.558135827Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.561930019Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.565455098Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"worker"}
{"msg":"region 106fca46-fef1-4b71-bb4b-7b85d6deea47 running snapshot 9e302fab-288d-4912-8d43-d6946fbfde66 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:22.565470878Z","hostname":"oxz_crucible_a3022a1c-6de3-4a49-88ef-adc23e4a7379","pid":25033,"component":"datafile"}