[ Jul 18 22:03:04 Enabled. ]
[ Jul 18 22:03:04 Rereading configuration. ]
[ Jul 18 22:03:06 Rereading configuration. ]
[ Jul 18 22:03:08 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9317dd26-3ead-4d57-b6b1-e3986c5dd74c/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9317dd26-3ead-4d57-b6b1-e3986c5dd74c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:08.853975111Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:08.855404221Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:08.855431251Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:08.879787468Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:09.01698905Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.28778797Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:41144"}
{"msg":"region cc662575-690d-4143-a67b-4f4546be960a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.313525436Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.320651968Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"uri":"/crucible/0/regions","method":"POST","req_id":"6b45da1e-fabb-4908-8be1-d8cf728417a4","remote_addr":"[fd00:1122:3344:101::b]:41144","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":15397,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.320683788Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_9317dd26-3ead-4d57-b6b1-e3986c5dd74c/crucible/regions/cc662575-690d-4143-a67b-4f4546be960a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.334961501Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9317dd26-3ead-4d57-b6b1-e3986c5dd74c/crucible/regions/cc662575-690d-4143-a67b-4f4546be960a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.335032942Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"cc662575-690d-4143-a67b-4f4546be960a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cc662575-690d-4143-a67b-4f4546be960a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.481572267Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"region":"cc662575-690d-4143-a67b-4f4546be960a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cc662575-690d-4143-a67b-4f4546be960a\" \"--data\" \"/data/regions/cc662575-690d-4143-a67b-4f4546be960a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.488838631Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"region":"cc662575-690d-4143-a67b-4f4546be960a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.672004117Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"uri":"/crucible/0/regions","method":"POST","req_id":"8af8509e-17d8-4737-9a98-44e964515786","remote_addr":"[fd00:1122:3344:101::b]:41144","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.797691376Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"region":"cc662575-690d-4143-a67b-4f4546be960a","component":"worker"}
{"msg":"region cc662575-690d-4143-a67b-4f4546be960a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.797776477Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.798042271Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cc662575-690d-4143-a67b-4f4546be960a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.798540488Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cc662575-690d-4143-a67b-4f4546be960a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.81762658Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.841947026Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cc662575-690d-4143-a67b-4f4546be960a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.853676923Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cc662575-690d-4143-a67b-4f4546be960a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.854456224Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.858924207Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.859019029Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.859064609Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.886365648Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:35.911408734Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:36.285134402Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"uri":"/crucible/0/regions","method":"POST","req_id":"e087155c-4a38-4315-af9f-e98914926b67","remote_addr":"[fd00:1122:3344:101::b]:41144","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.53517368Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"uri":"/crucible/0/regions/cc662575-690d-4143-a67b-4f4546be960a","method":"GET","req_id":"5c1688c8-58c9-44f0-aff8-0439d6697b85","remote_addr":"[fd00:1122:3344:101::b]:41144","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"path is \"/data/regions/cc662575-690d-4143-a67b-4f4546be960a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.537100117Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"datafile"}
{"msg":"dataset is oxp_9317dd26-3ead-4d57-b6b1-e3986c5dd74c/crucible/regions/cc662575-690d-4143-a67b-4f4546be960a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.546777575Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"datafile"}
{"msg":"region cc662575-690d-4143-a67b-4f4546be960a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.554931391Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.555118014Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.555182115Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"uri":"/crucible/0/regions/cc662575-690d-4143-a67b-4f4546be960a","method":"DELETE","req_id":"9b6ace0d-aa97-42db-b1a9-2a3f9115f439","remote_addr":"[fd00:1122:3344:101::b]:41144","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18143,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-cc662575-690d-4143-a67b-4f4546be960a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.55626367Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.557112172Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"uri":"/crucible/0/regions/cc662575-690d-4143-a67b-4f4546be960a","method":"GET","req_id":"3df44989-5ca6-44c6-9645-e8eda6f6d6e3","remote_addr":"[fd00:1122:3344:101::b]:41144","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.570386971Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9317dd26-3ead-4d57-b6b1-e3986c5dd74c/crucible/regions/cc662575-690d-4143-a67b-4f4546be960a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.578810921Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"region":"cc662575-690d-4143-a67b-4f4546be960a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.634877459Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"uri":"/crucible/0/regions/cc662575-690d-4143-a67b-4f4546be960a","method":"GET","req_id":"377789fa-e7f1-4595-bcef-1d63090845de","remote_addr":"[fd00:1122:3344:101::b]:41144","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region cc662575-690d-4143-a67b-4f4546be960a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.67920204Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-18T22:03:37.815205465Z","hostname":"oxz_crucible_f73eceba-cae3-4f1c-a312-f4ad2539646f","pid":24804,"uri":"/crucible/0/regions/cc662575-690d-4143-a67b-4f4546be960a","method":"GET","req_id":"f1c25baf-5c9f-4397-9581-6453d2ec4103","remote_addr":"[fd00:1122:3344:101::b]:41144","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26,"response_code":"200"}