Question

I am trying to recombine / push an array called $new_datas assigned to one json file with the $old_datas assigned to another json file.

I have tried to merge the two, but with the limitation of 100 items by pull/for each request on an external Api, sometimes when I use the offset for this Api some duplicate content exists on the final json file. I try to unset the value before pushing the $new_datas, but currently the duplicate contents already exists.

What is the best way to do this? I’m just an enthusiast, currently not a professional so I have started studying only a month ago. Thanks in advance for your help

There is my code

    //Offset twitch

    $next_offset = get_option('mmob_next_offset');

        //First run
        if(!$next_offset)
        {
            $next_offset = '0';
        }

        $query_api_link_twitch = 'http://api.justin.tv/api/stream/list.json?limit=99&category=gaming&offset='.$next_offset.''; // /!\ json

        //Get all contents
        $datas_get_twitch = file_get_contents($query_api_link_twitch);

        if(!empty($datas_get_twitch))
        {
            $new_datas_twitch = json_decode($datas_get_twitch);
            $old_datas_twitch = json_decode(file_get_contents($cache_target . 'twitch.json'));

            $olds = array();

                    //I have tried this function before the array_merge but don't work
                    //I try to compare the 2nd value from the key "name" because it's the only one "unique"
            foreach($old_datas_twitch as $key => $value)
            {
                //var_dump($value);

                $olds[] = $value->name;

                //var_dump($name);

                foreach($new_datas_twitch as $k => $v)
                {
                    $new = $v->name;

                    if(in_array($new, $olds))
                    {
                        unset($old_datas_twitch->$key);
                    }
                }
            }

            //Merge, put, close
            $twitch_new_build = array_merge((array)$old_datas_twitch, (array)$new_datas_twitch);
            $datas_push_twitch = file_put_contents($cache_target . 'twitch.json', json_encode($twitch_new_build));

            update_option('mmob_next_offset', $next_offset + 100);

if(!file_exists($cache_target . 'twitch.json') || !is_writable($cache_target . 'twitch.json'))
        {
            $email = false;

            $last_known_day = (int)date('d', get_option('mmob_email_date_twitch'));
            $current_day = (int)date('d', time());

            //Cron run every 2 minutes...
            if($current_day > $last_known_day)
            {
                $email = true;
            }

            update_option('mmob_email_date_twitch', time());
        }
    }
    else
    {
        update_option('mmob_next_offset', '0');
    }

There is the var_dump

object(stdClass)[2533]
  public 'embed_count' => string '0' (length=1)
  public 'name' => string 'live_user_lunaticoldschool' (length=26)
  public 'stream_count' => string '1' (length=1)
  public 'subcategory' => string 'strategy' (length=8)
  public 'category' => string 'gaming' (length=6)
  public 'format' => string 'live' (length=4)
  public 'channel_count' => int 1
  public 'title' => string '"LuNaTiC Teh Macro Zerg" HD Streaming StarCraft 2 EU-Server playing Zerg [Season 4 - 2012 @ Gold League, Some Fun Random Team Games Also]  ' (length=139)
  public 'featured' => boolean true
  public 'site_count' => string '1' (length=1)
  public 'abuse_reported' => boolean false
  public 'channel' => 
    object(stdClass)[2534]
      public 'embed_enabled' => boolean true
      public 'language' => string 'en' (length=2)
      public 'views_count' => string '771554' (length=6)
      public 'embed_code' => string '    <object type="application/x-shockwave-flash" height="295" width="353" id="live_embed_player_flash" data="http://www.justin.tv/widgets/live_embed_player.swf?channel=lunaticoldschool" bgcolor="#000000"><param name="allowFullScreen" value="true" /><param name="allowscriptaccess" value="always" /><param name="movie" value="http://www.justin.tv/widgets/live_embed_player.swf" /><param name="flashvars" value="auto_play=false&channel=lunaticoldschool&start_volume=25" /></object>
' (length=480)
      public 'screen_cap_url_small' => string 'http://static-cdn.jtvnw.net/previews/live_user_lunaticoldschool-70x53.jpg' (length=73)
      public 'subcategory' => string 'strategy' (length=8)
      public 'category' => string 'gaming' (length=6)
      public 'status' => string '"LuNaTiC Teh Macro Zerg" HD Streaming StarCraft 2 EU-Server playing Zerg [Season 4 - 2012 @ Gold League, Some Fun Random Team Games Also]  ' (length=139)
      public 'image_url_huge' => string 'http://static-cdn.jtvnw.net/jtv_user_pictures/lunaticoldschool-profile_image-c283f71a8088b0a3-600x600.jpeg' (length=106)
      public 'mature' => boolean false
      public 'producer' => boolean true
      public 'image_url_tiny' => string 'http://static-cdn.jtvnw.net/jtv_user_pictures/lunaticoldschool-profile_image-c283f71a8088b0a3-50x50.jpeg' (length=104)
      public 'title' => string 'LuNaTiC Teh Macro Zerg ' (length=23)
      public 'image_url_large' => string 'http://static-cdn.jtvnw.net/jtv_user_pictures/lunaticoldschool-profile_image-c283f71a8088b0a3-300x300.jpeg' (length=106)
      public 'category_title' => string 'Gaming' (length=6)
      public 'id' => int 19437762
      public 'screen_cap_url_huge' => string 'http://static-cdn.jtvnw.net/previews/live_user_lunaticoldschool-630x473.jpg' (length=75)
      public 'image_url_small' => string 'http://static-cdn.jtvnw.net/jtv_user_pictures/lunaticoldschool-profile_image-c283f71a8088b0a3-70x70.jpeg' (length=104)
      public 'image_url_medium' => string 'http://static-cdn.jtvnw.net/jtv_user_pictures/lunaticoldschool-profile_image-c283f71a8088b0a3-150x150.jpeg' (length=106)
      public 'timezone' => string 'Europe/Athens' (length=13)
      public 'subcategory_title' => string 'Strategy' (length=8)
      public 'screen_cap_url_large' => string 'http://static-cdn.jtvnw.net/previews/live_user_lunaticoldschool-320x240.jpg' (length=75)
      public 'channel_url' => string 'http://www.justin.tv/lunaticoldschool' (length=37)
      public 'tags' => null
      public 'login' => string 'lunaticoldschool' (length=16)
      public 'screen_cap_url_medium' => string 'http://static-cdn.jtvnw.net/previews/live_user_lunaticoldschool-150x113.jpg' (length=75)
  public 'video_height' => int 720
  public 'language' => string 'en' (length=2)
  public 'video_bitrate' => float 309.2265625
  public 'id' => string '4010553696' (length=10)
  public 'meta_game' => string 'StarCraft II: Wings of Liberty' (length=30)
  public 'broadcaster' => string 'fme' (length=3)
  public 'broadcast_part' => int 1
  public 'audio_codec' => string 'aac' (length=3)
  public 'up_time' => string 'Fri Oct 19 06:18:15 2012' (length=24)
  public 'video_width' => int 1280
  public 'geo' => string 'GR' (length=2)
  public 'channel_view_count' => int 771541
  public 'channel_subscription' => boolean false
  public 'embed_enabled' => boolean true
  public 'stream_type' => string 'live' (length=4)
  public 'video_codec' => string 'AVC' (length=3)

Answer 1

What you’re look to do is create a set from two sets (or maybe lists, but my guess is sets). To do this you need to decide what the unique id is going to be.

In my first example the unique_id is simply the key of the objects being passed. In my second example it is a string created by concatenating multiple fields from the item.

There is a possible third example that takes the unique_id creation used in example 2 and the overwrite functionality used in example 1, but this will return an array keyed by your unique_id so it might not be desired.

Example One: Overwrite Keyed Array

function example_one_overwrite($old, $new){
    //THIS METHOD ASSUMES THAT KEY IS YOUR UNIQUE ID AND THAT YOU WANT A KEYED ARRAY RETURNED
    $unique = array();

    //DATA THAT MIGHT BE POSSIBLY OVERWRITTEN
    foreach($old as $key=>$item){
        $unique[$key] = $item;
    }

    //DATA THAT MIGHT POSSIBLY OVERWRITE OLD DATA
    foreach($new as $key=>$item){
        $unique[$key] = $item;
    }

    return $unique;
}

Example Two: Single Write to Array, Numeric Keys

function example_two_singlewrite($old, $new){

    //This method assumes you want a numeric array when you are done, not an associative one. 

    $unique = array();
    $used_unique_ids = array();

    //KEEP DATA
    foreach($new as $item){

        //I have this set to $item->name cause I don't know what you want to use to consider something unique
        //But another example might be $unique_id = $item->name.'-'.$item->format;

        $unique_id = $item->name;

        if(!isset($used_unique_ids[$unique_id])){
            $used_unique_ids[$unique_id] = "used";
            $unique[] = $item;
        }
    }

    //DATA TO KEEP ONLY IF IT WAS NOT IN THE OTHER LIST
    foreach($list as $item){

        $unique_id = $item->name;

        if(!isset($used_unique_ids[$unique_id])){
            $used_unique_ids[$unique_id] = "used";
            $unique[] = $item;
        }
    }

    return $unique;
}

HOW TO USE THESE EXAMPLES IN YOUR CODE

//NOTE: I'm not sure if you need the $key in your result, if so mod the example_two_singlewrite function.

//STARTING AFTER $old_datas_twitch = json_decode...
//REPLACE ALL CODE TILL $datas_push_twitch = file_put... WITH
$twitch_new_build = example_two_singlewrite($old_datas_twitch , $new_datas_twitch);

//ADD FUNCTION example_two_singlewrite to your file