[ Feb 2 02:33:15 Enabled. ] [ Feb 2 02:33:15 Rereading configuration. ] [ Feb 2 02:33:18 Rereading configuration. ] [ Feb 2 02:33:23 Executing start method ("/opt/oxide/lib/svc/manifest/crucible/agent.sh"). ] + . /lib/svc/share/smf_include.sh ++ SMF_EXIT_OK=0 ++ SMF_EXIT_NODAEMON=94 ++ SMF_EXIT_ERR_FATAL=95 ++ SMF_EXIT_ERR_CONFIG=96 ++ SMF_EXIT_MON_DEGRADE=97 ++ SMF_EXIT_MON_OFFLINE=98 ++ SMF_EXIT_ERR_NOSMF=99 ++ SMF_EXIT_ERR_PERM=100 ++ svcprop -c -p config/datalink svc:/oxide/crucible/agent:default + DATALINK=oxControlService15 ++ svcprop -c -p config/gateway svc:/oxide/crucible/agent:default + GATEWAY=fd00:1122:3344:101::1 ++ svcprop -c -p config/dataset svc:/oxide/crucible/agent:default + DATASET=oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible ++ svcprop -c -p config/listen_addr svc:/oxide/crucible/agent:default + LISTEN_ADDR=fd00:1122:3344:101::16 ++ svcprop -c -p config/listen_port svc:/oxide/crucible/agent:default + LISTEN_PORT=32345 ++ svcprop -c -p config/portbase svc:/oxide/crucible/agent:default + PORTBASE=19000 ++ svcprop -c -p config/downstairs_prefix svc:/oxide/crucible/agent:default + DOWNSTAIRS_PREFIX=downstairs ++ svcprop -c -p config/snapshot_prefix svc:/oxide/crucible/agent:default + SNAPSHOT_PREFIX=snapshot + [[ oxControlService15 == unknown ]] + [[ fd00:1122:3344:101::1 == unknown ]] + ipadm delete-if oxControlService15 ipadm: Could not delete oxControlService15: Interface does not exist + true + ipadm create-if -t oxControlService15 + ipadm set-ifprop -t -p mtu=9000 -m ipv4 oxControlService15 + ipadm set-ifprop -t -p mtu=9000 -m ipv6 oxControlService15 + ipadm show-addr oxControlService15/ll ipadm: Address object not found + ipadm create-addr -t -T addrconf oxControlService15/ll + ipadm show-addr oxControlService15/omicron6 ipadm: Address object not found + ipadm create-addr -t -T static -a fd00:1122:3344:101::16 oxControlService15/omicron6 + route get -inet6 default -inet6 fd00:1122:3344:101::1 default: not in table + route add -inet6 default -inet6 fd00:1122:3344:101::1 add net default: gateway fd00:1122:3344:101::1 + args=('-D' '/opt/oxide/crucible/bin/crucible-downstairs' '--dataset' "$DATASET" '-l' "[$LISTEN_ADDR]:$LISTEN_PORT" '-P' "$PORTBASE" '-p' "$DOWNSTAIRS_PREFIX" '-s' "$SNAPSHOT_PREFIX") + exec /opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible -l '[fd00:1122:3344:101::16]:32345' -P 19000 -p downstairs -s snapshot note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:33:24.354011077Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632} {"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:33:24.380647812Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:33:24.380659902Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:33:24.747809842Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:27.726014617Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58886"} {"msg":"region e4cfe51f-c926-4d1a-a47a-20a931173f1a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:27.790173515Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:27.790401731Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions","method":"POST","req_id":"052c50af-5a0e-47d7-898c-15666581ab6a","remote_addr":"[fd00:1122:3344:101::a]:58886","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":1266,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:27.794960113Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:27.804654212Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:27.804686292Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e4cfe51f-c926-4d1a-a47a-20a931173f1a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:27.864275108Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"region":"e4cfe51f-c926-4d1a-a47a-20a931173f1a","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:27.981391679Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions","method":"POST","req_id":"30663b01-2452-4138-ad90-74f74de69513","remote_addr":"[fd00:1122:3344:101::a]:58886","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":73,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.208410242Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"region":"e4cfe51f-c926-4d1a-a47a-20a931173f1a","component":"worker"} {"msg":"region e4cfe51f-c926-4d1a-a47a-20a931173f1a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.208455553Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.208560856Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e4cfe51f-c926-4d1a-a47a-20a931173f1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.208982806Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e4cfe51f-c926-4d1a-a47a-20a931173f1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.216791818Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.229017349Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e4cfe51f-c926-4d1a-a47a-20a931173f1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.234881263Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.23558767Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.235712804Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.235760485Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.235801486Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.242169903Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.247967355Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:34:28.349292327Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions","method":"POST","req_id":"c8915b91-4906-49e9-b1f6-68009fdc8833","remote_addr":"[fd00:1122:3344:101::a]:58886","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.272538978Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45371"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.272653581Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a","method":"GET","req_id":"f1d0bbb9-d51b-4b79-9516-a9ed2a1404a7","remote_addr":"[fd00:1122:3344:101::a]:45371","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":75,"response_code":"200"} {"msg":"path is \"/data/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.280123495Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.289571927Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"snapshot is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a@043d67e6-7244-453f-b2d4-b5c90f72c8d3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.298985879Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a\", \"043d67e6-7244-453f-b2d4-b5c90f72c8d3\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.29902388Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.308325178Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a/snapshots/043d67e6-7244-453f-b2d4-b5c90f72c8d3","method":"GET","req_id":"ec0e248b-16b9-4fc6-9335-dd6190fbcd10","remote_addr":"[fd00:1122:3344:101::a]:45371","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":35499,"response_code":"200"} {"msg":"path is \"/data/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.308518953Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.317245258Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"snapshot is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a@043d67e6-7244-453f-b2d4-b5c90f72c8d3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.326067925Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a\", \"043d67e6-7244-453f-b2d4-b5c90f72c8d3\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.326097776Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot e4cfe51f-c926-4d1a-a47a-20a931173f1a-043d67e6-7244-453f-b2d4-b5c90f72c8d3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.336868141Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"applying SMF actions for region e4cfe51f-c926-4d1a-a47a-20a931173f1a running snapshot 043d67e6-7244-453f-b2d4-b5c90f72c8d3 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.337475475Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.337529677Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a/snapshots/043d67e6-7244-453f-b2d4-b5c90f72c8d3/run","method":"POST","req_id":"e3eeec10-ed96-46ca-86b8-bcb0be8a1bbd","remote_addr":"[fd00:1122:3344:101::a]:45371","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":29037,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-e4cfe51f-c926-4d1a-a47a-20a931173f1a-043d67e6-7244-453f-b2d4-b5c90f72c8d3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.34336228Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e4cfe51f-c926-4d1a-a47a-20a931173f1a-043d67e6-7244-453f-b2d4-b5c90f72c8d3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.349016269Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e4cfe51f-c926-4d1a-a47a-20a931173f1a-043d67e6-7244-453f-b2d4-b5c90f72c8d3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.368283363Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e4cfe51f-c926-4d1a-a47a-20a931173f1a/.zfs/snapshot/043d67e6-7244-453f-b2d4-b5c90f72c8d3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.368529789Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.36858287Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.368628262Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.368671163Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.368723514Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.375272635Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.382060932Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"region e4cfe51f-c926-4d1a-a47a-20a931173f1a running snapshot 043d67e6-7244-453f-b2d4-b5c90f72c8d3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:06.382073742Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:08.895321586Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43854"} {"msg":"region cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:08.95685454Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:08.957016774Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:08.957027794Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions","method":"POST","req_id":"a0c9c3c3-ec1b-4126-b8fb-6afef5c3e7ae","remote_addr":"[fd00:1122:3344:101::a]:43854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":227,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:08.966917427Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:08.966949638Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.045445089Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"region":"cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.143338776Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"region":"cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","component":"worker"} {"msg":"region cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.143387808Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.14347563Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.14390105Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.150520263Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.16218784Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.164343423Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions","method":"POST","req_id":"af9872c2-4faf-46d4-bd6d-a438c8ee6895","remote_addr":"[fd00:1122:3344:101::a]:43854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":65,"response_code":"200"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.167096091Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.167347027Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.167397618Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.167439729Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.16748239Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.174592995Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:09.190610799Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.231504132Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54379"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.231765788Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","method":"GET","req_id":"cb813a6f-c262-44a5-bb1f-ab9ce7c799ba","remote_addr":"[fd00:1122:3344:101::a]:54379","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":157,"response_code":"200"} {"msg":"path is \"/data/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.231973323Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.244989303Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"region cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.253592755Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.253755339Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.2538135Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","method":"DELETE","req_id":"0065db88-eff2-444d-9115-6721175a16e2","remote_addr":"[fd00:1122:3344:101::a]:54379","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21888,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.253981585Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","method":"GET","req_id":"4d9a0d92-5c79-4bb2-82a4-b74ed99f0d02","remote_addr":"[fd00:1122:3344:101::a]:54379","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":30,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.254831475Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.261782096Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.272593792Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"region":"cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","component":"worker"} {"msg":"region cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.319293551Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-02T02:40:40.324369796Z","hostname":"oxz_crucible_670beb6f-4fc9-4fed-b03f-652a8d62fc10","pid":22632,"uri":"/crucible/0/regions/cc6a4f86-e5d2-445a-8886-0d0d7d6b65ea","method":"GET","req_id":"8e7bc3f1-eb33-40f4-a301-325c340ade57","remote_addr":"[fd00:1122:3344:101::a]:54379","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":86,"response_code":"200"}