scrape beach with PHP and Supabase

Hasilnya

Working on CMD

how to run?

php loop.php
// atau
php loop.php --halaman=109

simpan dengan nama "loop.php"

<?php

// Set time limit ke 0 untuk mencegah timeout
set_time_limit(0);


$supabaseUrl = 'https://xxx.supabase.co';
$supabaseKey = 'xxxxxxxxx';

$apiKey = 'xxxxxxxxxxxxxxxxx';
$radius = 50000; // Radius pencarian dalam meter
$type = 'natural_feature'; // Tipe lokasi
$keyword = 'pantai';



$kota = [           
  "Banjar, Jawa Barat, Indonesia",
  "Banjar, Kalimantan Selatan, Indonesia",
  "Banjarbaru, Kalimantan Selatan, Indonesia",
  "Banjarmasin, Kalimantan Selatan, Indonesia",
  "Banjarnegara, Jawa Tengah, Indonesia",
  "Bantaeng, Sulawesi Selatan, Indonesia",
  "Way Kanan, Lampung, Indonesia",
  "Wonogiri, Jawa Tengah, Indonesia",
  "Wonosobo, Jawa Tengah, Indonesia",
  "Yahukimo, Papua, Indonesia",
  "Yalimo, Papua, Indonesia",
  "Yogyakarta, Daerah Istimewa Yogyakarta, Indonesia"
];

// Ambil nilai halaman dari command line argument
$options = getopt("", ["halaman:"]);
$halaman = isset($options['halaman']) ? intval($options['halaman']) : 0; // Default ke halaman 0 jika tidak ada parameter

echo "\033[34m -------------------------------------------------\033[0m\n";
echo "\033[34m Kota di Jawa Timur (Halaman " . $halaman + 1 . "): dari " . count($kota) . " Halaman \033[0m\n";
echo "\033[34m -------------------------------------------------\033[0m\n";

echo "\033[34m Untuk Memulai dari halaman tertentu, jalankan perintah: \033[0m\n";
echo "   php loop.php --halaman=3 \n";
echo "\033[34m -------------------------------------------------\033[0m\n";
echo "   \n";
echo "   \n";


// Tampilkan data berdasarkan halaman
if ($halaman >= 0 && $halaman < count($kota)) {


    function fetchPlaces($url, $apiKey) {
        $places = [];
        do {
            $response = file_get_contents($url);
            $data = json_decode($response, true);

            if (!empty($data['results'])) {
                $places = array_merge($places, $data['results']);
            }

            $nextPageToken = isset($data['next_page_token']) ? $data['next_page_token'] : null;
            if ($nextPageToken) {
                sleep(2);
                $url = "https://maps.googleapis.com/maps/api/place/nearbysearch/json?pagetoken=$nextPageToken&key=$apiKey";
            }
        } while ($nextPageToken);

        return $places;
    }

    function fetchPlaceDetails($placeId, $apiKey) {
        $url = "https://maps.googleapis.com/maps/api/place/details/json?place_id=$placeId&key=$apiKey";
        $response = file_get_contents($url);
        return json_decode($response, true);
    }

    # BEGIN loop scrape here


    $hal = $halaman -1;
    if($hal <= 0) { $hal = 0; }

    for ($i = $hal; $i < count($kota); $i++) {
      echo ($i+1).'. '.$kota[$i] . "\n";

      $kotanya = '';
      $kotanya = $kota[$i];

#beg

      if (!empty($kotanya)) {
          $locationName = urlencode($kotanya);

          // Menggunakan API Geocoding untuk mendapatkan koordinat dari nama lokasi
          $geoUrl = "https://maps.googleapis.com/maps/api/geocode/json?address=$locationName&key=$apiKey";
          $geoResponse = file_get_contents($geoUrl);
          $geoData = json_decode($geoResponse, true);

          if (!empty($geoData['results'][0]['geometry']['location'])) {
              $lat = $geoData['results'][0]['geometry']['location']['lat'];
              $lng = $geoData['results'][0]['geometry']['location']['lng'];
              $location = "$lat,$lng";


              $url = "https://maps.googleapis.com/maps/api/place/nearbysearch/json?location=$location&radius=$radius&type=$type&keyword=$keyword&key=$apiKey";
              $places = fetchPlaces($url, $apiKey);

              if (!empty($places)) {



                  $headers = [
                      "Content-Type: application/json",
                      "Authorization: Bearer $supabaseKey",
                      "apikey: $supabaseKey"
                  ];

                  $supabaseInsertUrl = "$supabaseUrl/rest/v1/places";

                  foreach ($places as $place) {
                      $name = $place['name'];
                      $lat = $place['geometry']['location']['lat'];
                      $lng = $place['geometry']['location']['lng'];
                      $placeId = $place['place_id'];
                      $rating = isset($place['rating']) ? $place['rating'] : null;
                      $vicinity = isset($place['vicinity']) ? $place['vicinity'] : null;
                      $types = isset($place['types']) ? json_encode($place['types']) : json_encode([]);

                      $details = fetchPlaceDetails($placeId, $apiKey);
                      $formattedAddress = isset($details['result']['formatted_address']) ? $details['result']['formatted_address'] : null;

                      $reviews = [];
                      if (isset($details['result']['reviews'])) {
                          foreach ($details['result']['reviews'] as $review) {
                              if (isset($review['text'])) {
                                  $reviews[] = htmlspecialchars($review['text']);
                              }
                          }
                      }

                      $photoUrls = [];
                      if (isset($details['result']['photos'])) {
                          foreach ($details['result']['photos'] as $photo) {
                              if (isset($photo['photo_reference'])) {
                                  $photoUrls[] = "https://maps.googleapis.com/maps/api/place/photo?maxwidth=400&photoreference=" . $photo['photo_reference'] . "&key=$apiKey";
                              }
                          }
                      }

                      $postData = json_encode([
                          "name" => $name,
                          "latitude" => $lat,
                          "longitude" => $lng,
                          "place_id" => $placeId,
                          "rating" => $rating,
                          "formatted_address" => $formattedAddress,
                          "types" => $types,
                          "vicinity" => $vicinity,
                          "reviews" => json_encode($reviews),
                          "photo_urls" => json_encode($photoUrls)
                      ]);

                      $ch = curl_init();
                      curl_setopt($ch, CURLOPT_URL, $supabaseInsertUrl);
                      curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
                      curl_setopt($ch, CURLOPT_POST, true);
                      curl_setopt($ch, CURLOPT_POSTFIELDS, $postData);
                      curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

                      $response = curl_exec($ch);
                      curl_close($ch);
                  }

                  echo "\033[32m-------------------------------------------------\033[0m\n";
                  echo "\033[32m--  [ BERHASIL ] ". $kota[$i] ."  \033[0m\n";
                  echo "\033[32m-------------------------------------------------\033[0m\n";

              } else {
                  echo "\033[31m-------------------------------------------------\033[0m\n";
                  echo "\033[31m -- Tidak ada hasil ditemukan -- \033[0m\n";
                  echo "\033[31m-------------------------------------------------\033[0m\n";

              }
          } else {
                  echo "\033[31m-------------------------------------------------\033[0m\n";
                  echo "\033[31m -- Lokasi tidak ditemukan -- \033[0m\n";
                  echo "\033[31m-------------------------------------------------\033[0m\n";
          }
      } 
#end


      if (($i+1) == count($kota)) {
          echo "\033[34m-------------------------------------------------\033[0m\n";
          echo "\033[34m Ini adalah halaman terakhir \033[0m\n";
          echo "\033[34m-------------------------------------------------\033[0m\n";
      }

    }

    # END scrape here



} else {
    echo "\033[31m-------------------------------------------------\033[0m\n";
    echo "\033[31m -- Tidak ada data untuk halaman ini -- \033[0m\n";
    echo "\033[31m-------------------------------------------------\033[0m\n";
}

?>

Ya begitulah