[ Dec 14 04:34:55 Enabled. ]
[ Dec 14 04:34:55 Rereading configuration. ]
[ Dec 14 04:34:57 Rereading configuration. ]
[ Dec 14 04:35:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9b71970a-c525-4b2a-8e5d-7dd0f4e302f1/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9b71970a-c525-4b2a-8e5d-7dd0f4e302f1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:00.7461099Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:00.751979812Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:00.751994983Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:00.769250873Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:00.841772432Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:27.792491127Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37993"}
{"msg":"region cd9ed80b-23cd-4aac-99bb-ecf665b63900 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:27.835283232Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:27.840432026Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:27.847040178Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"33ac24e3-13d5-41e5-8039-7e59f059d51e","remote_addr":"[fd00:1122:3344:101::c]:37993","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":16967,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_9b71970a-c525-4b2a-8e5d-7dd0f4e302f1/crucible/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:27.853441995Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9b71970a-c525-4b2a-8e5d-7dd0f4e302f1/crucible/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:27.853488186Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"cd9ed80b-23cd-4aac-99bb-ecf665b63900\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.010103277Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"region":"cd9ed80b-23cd-4aac-99bb-ecf665b63900","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cd9ed80b-23cd-4aac-99bb-ecf665b63900\" \"--data\" \"/data/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.01020306Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"region":"cd9ed80b-23cd-4aac-99bb-ecf665b63900","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.013074615Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"4a88b05e-db3f-485a-8070-cf1983021769","remote_addr":"[fd00:1122:3344:101::c]:37993","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":555,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.296052048Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"01c4e528-6fce-4bc9-8a44-90c6835b1080","remote_addr":"[fd00:1122:3344:101::c]:37993","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":47,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.392982864Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"region":"cd9ed80b-23cd-4aac-99bb-ecf665b63900","component":"worker"}
{"msg":"region cd9ed80b-23cd-4aac-99bb-ecf665b63900 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.393072956Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.393309643Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cd9ed80b-23cd-4aac-99bb-ecf665b63900","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.393685772Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cd9ed80b-23cd-4aac-99bb-ecf665b63900","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.403655992Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.423298454Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cd9ed80b-23cd-4aac-99bb-ecf665b63900","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.429833724Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.430543302Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.430608284Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.430650955Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.430686046Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.441001725Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:28.454528867Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:29.748944525Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"d611dbed-f900-4284-85b0-eff76424a5e5","remote_addr":"[fd00:1122:3344:101::c]:37993","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.12143427Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"uri":"/crucible/0/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900","method":"GET","req_id":"ee2d92f9-b3ed-4edd-a826-46fcb024b737","remote_addr":"[fd00:1122:3344:101::c]:37993","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"path is \"/data/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.124781018Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"datafile"}
{"msg":"dataset is oxp_9b71970a-c525-4b2a-8e5d-7dd0f4e302f1/crucible/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.136893123Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"datafile"}
{"msg":"region cd9ed80b-23cd-4aac-99bb-ecf665b63900 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.148225198Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.148394263Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.148462685Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"uri":"/crucible/0/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900","method":"DELETE","req_id":"92b5d19d-e6f4-49bd-b25d-34100d822d8c","remote_addr":"[fd00:1122:3344:101::c]:37993","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":23728,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-cd9ed80b-23cd-4aac-99bb-ecf665b63900 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.152221662Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.152483509Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"uri":"/crucible/0/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900","method":"GET","req_id":"0bec7b7e-f225-407b-8130-b40790641f7d","remote_addr":"[fd00:1122:3344:101::c]:37993","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.159702477Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9b71970a-c525-4b2a-8e5d-7dd0f4e302f1/crucible/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.171873015Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"region":"cd9ed80b-23cd-4aac-99bb-ecf665b63900","component":"worker"}
{"msg":"region cd9ed80b-23cd-4aac-99bb-ecf665b63900 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.255170815Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-14T04:35:30.29679172Z","hostname":"oxz_crucible_9ed9cdcd-c0e6-404a-94b6-ea7b2bb38962","pid":23179,"uri":"/crucible/0/regions/cd9ed80b-23cd-4aac-99bb-ecf665b63900","method":"GET","req_id":"29955950-c407-4abc-85d7-d33bbf7c3960","remote_addr":"[fd00:1122:3344:101::c]:37993","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":71,"response_code":"200"}