[ Dec 14 01:03:05 Enabled. ]
[ Dec 14 01:03:05 Rereading configuration. ]
[ Dec 14 01:03:06 Rereading configuration. ]
[ Dec 14 01:03:12 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:03:12.240623816Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:03:12.24422967Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:03:12.244240111Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:03:12.260760789Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:03:12.416940918Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.344557074Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54694"}
{"msg":"region 129047c4-1eb7-4c91-b6ab-304f2d729140 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.385458228Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.390287488Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.395579691Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions","method":"POST","req_id":"cc62ac83-5778-458f-84fd-a22894528af2","remote_addr":"[fd00:1122:3344:101::c]:54694","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":17484,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/129047c4-1eb7-4c91-b6ab-304f2d729140","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.402201293Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/129047c4-1eb7-4c91-b6ab-304f2d729140","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.402226203Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"129047c4-1eb7-4c91-b6ab-304f2d729140\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/129047c4-1eb7-4c91-b6ab-304f2d729140\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.471518699Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"129047c4-1eb7-4c91-b6ab-304f2d729140","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"129047c4-1eb7-4c91-b6ab-304f2d729140\" \"--data\" \"/data/regions/129047c4-1eb7-4c91-b6ab-304f2d729140\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.473806455Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"129047c4-1eb7-4c91-b6ab-304f2d729140","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.625216816Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions","method":"POST","req_id":"02649018-cebc-4b45-8f00-ac959ff9c611","remote_addr":"[fd00:1122:3344:101::c]:54694","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.75698559Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"129047c4-1eb7-4c91-b6ab-304f2d729140","component":"worker"}
{"msg":"region 129047c4-1eb7-4c91-b6ab-304f2d729140 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.757028691Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.757123674Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-129047c4-1eb7-4c91-b6ab-304f2d729140","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.757507575Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-129047c4-1eb7-4c91-b6ab-304f2d729140","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.764147047Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.776067612Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-129047c4-1eb7-4c91-b6ab-304f2d729140","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.780467129Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/129047c4-1eb7-4c91-b6ab-304f2d729140","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.781218041Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.781281293Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.781330824Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.781372175Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.784888317Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.791019825Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:02.924627361Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions","method":"POST","req_id":"b735d63b-05a7-45e1-ac54-f9352f5b729d","remote_addr":"[fd00:1122:3344:101::c]:54694","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.416634869Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions/129047c4-1eb7-4c91-b6ab-304f2d729140","method":"GET","req_id":"42f95436-e428-4605-8569-9ec6812f5f66","remote_addr":"[fd00:1122:3344:101::c]:54694","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"path is \"/data/regions/129047c4-1eb7-4c91-b6ab-304f2d729140\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.419913044Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"dataset is oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/129047c4-1eb7-4c91-b6ab-304f2d729140","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.431377006Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"region 129047c4-1eb7-4c91-b6ab-304f2d729140 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.440960493Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.441100237Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.441178169Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions/129047c4-1eb7-4c91-b6ab-304f2d729140","method":"DELETE","req_id":"e8664011-a1b8-4609-8e13-c360680988d4","remote_addr":"[fd00:1122:3344:101::c]:54694","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21301,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.44466536Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions/129047c4-1eb7-4c91-b6ab-304f2d729140","method":"GET","req_id":"93890a15-30f1-4d44-8af4-44480061f37a","remote_addr":"[fd00:1122:3344:101::c]:54694","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":36,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-129047c4-1eb7-4c91-b6ab-304f2d729140 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.444796924Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.449818899Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/129047c4-1eb7-4c91-b6ab-304f2d729140\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.461793366Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"129047c4-1eb7-4c91-b6ab-304f2d729140","component":"worker"}
{"msg":"region 129047c4-1eb7-4c91-b6ab-304f2d729140 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.543021487Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:03.548232607Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions/129047c4-1eb7-4c91-b6ab-304f2d729140","method":"GET","req_id":"7168e97f-8cfb-44ab-835d-f08e1acc7cdc","remote_addr":"[fd00:1122:3344:101::c]:54694","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.41865554Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51700"}
{"msg":"region 46b81e51-3343-4191-bfe1-4bdf6fafca72 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.418859986Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.419046422Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.419055722Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions","method":"POST","req_id":"d9385a9c-ada6-4f08-9fb7-72303e3f62c1","remote_addr":"[fd00:1122:3344:101::a]:51700","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":229,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.427565598Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.427608259Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"46b81e51-3343-4191-bfe1-4bdf6fafca72\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.485979279Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"46b81e51-3343-4191-bfe1-4bdf6fafca72","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"46b81e51-3343-4191-bfe1-4bdf6fafca72\" \"--data\" \"/data/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.4860241Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"46b81e51-3343-4191-bfe1-4bdf6fafca72","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.56030103Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"46b81e51-3343-4191-bfe1-4bdf6fafca72","component":"worker"}
{"msg":"region 46b81e51-3343-4191-bfe1-4bdf6fafca72 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.560349061Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.560440603Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.561917796Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.564634795Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.574957904Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.577981301Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.578256479Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.578321071Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.578366162Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.578407223Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.582016818Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.586737884Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:04:33.643529258Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions","method":"POST","req_id":"ef5fc788-898d-48c3-ab6d-bfb5965b3eca","remote_addr":"[fd00:1122:3344:101::a]:51700","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.169743365Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47277"}
{"msg":"region 34abd9e6-046c-4a96-bd72-30087c37466c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.16990912Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.170425275Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.170439536Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions","method":"POST","req_id":"2f2fb526-fcd0-4b23-9598-22b6dcabdd96","remote_addr":"[fd00:1122:3344:101::a]:47277","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":588,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/34abd9e6-046c-4a96-bd72-30087c37466c","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.181441424Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/34abd9e6-046c-4a96-bd72-30087c37466c","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.181473755Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"34abd9e6-046c-4a96-bd72-30087c37466c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/34abd9e6-046c-4a96-bd72-30087c37466c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.240216535Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"34abd9e6-046c-4a96-bd72-30087c37466c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"34abd9e6-046c-4a96-bd72-30087c37466c\" \"--data\" \"/data/regions/34abd9e6-046c-4a96-bd72-30087c37466c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.240281817Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"34abd9e6-046c-4a96-bd72-30087c37466c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.358783926Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"region":"34abd9e6-046c-4a96-bd72-30087c37466c","component":"worker"}
{"msg":"region 34abd9e6-046c-4a96-bd72-30087c37466c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.358828957Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.35892311Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-34abd9e6-046c-4a96-bd72-30087c37466c","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.359898458Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-34abd9e6-046c-4a96-bd72-30087c37466c","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.370354261Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.383083919Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-34abd9e6-046c-4a96-bd72-30087c37466c","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.386099706Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/34abd9e6-046c-4a96-bd72-30087c37466c","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.386356324Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.386418555Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.386466567Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.386518258Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.389548766Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.394706235Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:49.440429018Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions","method":"POST","req_id":"37790f6f-6e8d-4d49-851a-ca0153f5d6bc","remote_addr":"[fd00:1122:3344:101::a]:47277","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":672,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.334076749Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72","method":"GET","req_id":"858eb5cd-ba49-4666-8cca-c7bd5517e0cd","remote_addr":"[fd00:1122:3344:101::a]:47277","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.337645072Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"dataset is oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.347274811Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"snapshot is oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72@2dfb33fe-d532-48ad-b085-3f8cd422800d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.356385135Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72\", \"2dfb33fe-d532-48ad-b085-3f8cd422800d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.356422085Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.368610918Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72/snapshots/2dfb33fe-d532-48ad-b085-3f8cd422800d","method":"GET","req_id":"71a61b3d-a0b7-4c07-91f9-a2e0c9615252","remote_addr":"[fd00:1122:3344:101::a]:47277","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":30995,"response_code":"200"}
{"msg":"path is \"/data/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.37178019Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"dataset is oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.379221745Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"snapshot is oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72@2dfb33fe-d532-48ad-b085-3f8cd422800d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.38800129Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72\", \"2dfb33fe-d532-48ad-b085-3f8cd422800d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.38803232Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 46b81e51-3343-4191-bfe1-4bdf6fafca72-2dfb33fe-d532-48ad-b085-3f8cd422800d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.396862346Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"applying SMF actions for region 46b81e51-3343-4191-bfe1-4bdf6fafca72 running snapshot 2dfb33fe-d532-48ad-b085-3f8cd422800d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.397045151Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.397079242Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72/snapshots/2dfb33fe-d532-48ad-b085-3f8cd422800d/run","method":"POST","req_id":"14accbf5-b224-4a31-b3d8-eb63df2f3c1e","remote_addr":"[fd00:1122:3344:101::a]:47277","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":25339,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-46b81e51-3343-4191-bfe1-4bdf6fafca72-2dfb33fe-d532-48ad-b085-3f8cd422800d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.399858893Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-46b81e51-3343-4191-bfe1-4bdf6fafca72-2dfb33fe-d532-48ad-b085-3f8cd422800d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.40218447Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-46b81e51-3343-4191-bfe1-4bdf6fafca72-2dfb33fe-d532-48ad-b085-3f8cd422800d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.410851631Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72/.zfs/snapshot/2dfb33fe-d532-48ad-b085-3f8cd422800d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.411108348Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.41117388Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.411223491Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.411266573Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.411323434Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.414719633Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.418902204Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"worker"}
{"msg":"region 46b81e51-3343-4191-bfe1-4bdf6fafca72 running snapshot 2dfb33fe-d532-48ad-b085-3f8cd422800d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.418926054Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"path is \"/data/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.72388047Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"dataset is oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.732512359Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"datafile"}
{"msg":"snapshot is oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72@2dfb33fe-d532-48ad-b085-3f8cd422800d","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.739914563Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f1a24525-2203-46b7-aea4-fc3971131d1e/crucible/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72\", \"2dfb33fe-d532-48ad-b085-3f8cd422800d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.739967635Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T01:14:50.748194163Z","hostname":"oxz_crucible_be4439cb-6cc7-4261-ab6e-3e59b46c7fc2","pid":22676,"uri":"/crucible/0/regions/46b81e51-3343-4191-bfe1-4bdf6fafca72/snapshots/2dfb33fe-d532-48ad-b085-3f8cd422800d/run","method":"POST","req_id":"705d3137-8445-4c05-8125-e97e5a39cf50","remote_addr":"[fd00:1122:3344:101::a]:47277","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24356,"response_code":"200"}